{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.999711953912626,
  "eval_steps": 500,
  "global_step": 2603,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.3646240234375,
      "epoch": 0.0003840614498319731,
      "grad_norm": 0.1572265625,
      "learning_rate": 3.831417624521072e-09,
      "loss": 0.0717,
      "num_tokens": 185747.0,
      "reward": 0.3359375074505806,
      "reward_std": 0.07373346667736769,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1848.8229370117188,
      "epoch": 0.0007681228996639462,
      "grad_norm": 0.1669921875,
      "learning_rate": 7.662835249042145e-09,
      "loss": 0.0376,
      "num_tokens": 377410.0,
      "reward": 0.3463541716337204,
      "reward_std": 0.10571773815900087,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3255208283662796,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.9167175292969,
      "epoch": 0.0011521843494959193,
      "grad_norm": 0.189453125,
      "learning_rate": 1.1494252873563218e-08,
      "loss": 0.0992,
      "num_tokens": 559442.0,
      "reward": 0.447916679084301,
      "reward_std": 0.18808503076434135,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3437499925494194,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.5521240234375,
      "epoch": 0.0015362457993278924,
      "grad_norm": 0.1826171875,
      "learning_rate": 1.532567049808429e-08,
      "loss": 0.0506,
      "num_tokens": 731695.0,
      "reward": 0.5494791865348816,
      "reward_std": 0.1724237408488989,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1738.0208740234375,
      "epoch": 0.0019203072491598655,
      "grad_norm": 0.1748046875,
      "learning_rate": 1.915708812260536e-08,
      "loss": 0.0764,
      "num_tokens": 914921.0,
      "reward": 0.4401041865348816,
      "reward_std": 0.16212281957268715,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671874925494194,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.4896240234375,
      "epoch": 0.0023043686989918386,
      "grad_norm": 0.1669921875,
      "learning_rate": 2.2988505747126436e-08,
      "loss": 0.057,
      "num_tokens": 1098296.0,
      "reward": 0.3619791716337204,
      "reward_std": 0.059532980900257826,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1677.2708740234375,
      "epoch": 0.0026884301488238117,
      "grad_norm": 0.2041015625,
      "learning_rate": 2.6819923371647508e-08,
      "loss": 0.0826,
      "num_tokens": 1274730.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.14043443836271763,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1840.1250610351562,
      "epoch": 0.0030724915986557848,
      "grad_norm": 0.1611328125,
      "learning_rate": 3.065134099616858e-08,
      "loss": 0.0348,
      "num_tokens": 1465374.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.18379579856991768,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291716337204,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.9688110351562,
      "epoch": 0.003456553048487758,
      "grad_norm": 0.1953125,
      "learning_rate": 3.448275862068965e-08,
      "loss": 0.053,
      "num_tokens": 1638331.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.19901340082287788,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.4166870117188,
      "epoch": 0.003840614498319731,
      "grad_norm": 0.220703125,
      "learning_rate": 3.831417624521072e-08,
      "loss": 0.0513,
      "num_tokens": 1814387.0,
      "reward": 0.393229179084301,
      "reward_std": 0.1122285476885736,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1502.2083740234375,
      "epoch": 0.0042246759481517045,
      "grad_norm": 0.244140625,
      "learning_rate": 4.21455938697318e-08,
      "loss": 0.0539,
      "num_tokens": 1977407.0,
      "reward": 0.5963541865348816,
      "reward_std": 0.20135986804962158,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.8958740234375,
      "epoch": 0.004608737397983677,
      "grad_norm": 0.25,
      "learning_rate": 4.597701149425287e-08,
      "loss": 0.0939,
      "num_tokens": 2132797.0,
      "reward": 0.518229179084301,
      "reward_std": 0.2102234922349453,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.2396240234375,
      "epoch": 0.004992798847815651,
      "grad_norm": 0.2177734375,
      "learning_rate": 4.9808429118773944e-08,
      "loss": 0.1362,
      "num_tokens": 2311460.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.1715594083070755,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.3854675292969,
      "epoch": 0.005376860297647623,
      "grad_norm": 0.1845703125,
      "learning_rate": 5.3639846743295015e-08,
      "loss": 0.0822,
      "num_tokens": 2497329.0,
      "reward": 0.401041679084301,
      "reward_std": 0.15501808654516935,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1525.1979370117188,
      "epoch": 0.005760921747479597,
      "grad_norm": 0.2177734375,
      "learning_rate": 5.747126436781609e-08,
      "loss": 0.0571,
      "num_tokens": 2657244.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.17037195153534412,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1359.4271240234375,
      "epoch": 0.0061449831973115696,
      "grad_norm": 0.2099609375,
      "learning_rate": 6.130268199233716e-08,
      "loss": 0.0701,
      "num_tokens": 2802245.0,
      "reward": 0.661458358168602,
      "reward_std": 0.2764439135789871,
      "rewards/accuracy_reward": 0.1979166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416641831398,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1975.2083740234375,
      "epoch": 0.006529044647143543,
      "grad_norm": 0.154296875,
      "learning_rate": 6.513409961685822e-08,
      "loss": 0.0431,
      "num_tokens": 3006681.0,
      "reward": 0.3281250149011612,
      "reward_std": 0.09319279715418816,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3281249925494194,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.8542175292969,
      "epoch": 0.006913106096975516,
      "grad_norm": 0.216796875,
      "learning_rate": 6.89655172413793e-08,
      "loss": 0.0977,
      "num_tokens": 3189379.0,
      "reward": 0.4479166716337204,
      "reward_std": 0.22044021263718605,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3750000074505806,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1512.0937805175781,
      "epoch": 0.007297167546807489,
      "grad_norm": 0.205078125,
      "learning_rate": 7.279693486590038e-08,
      "loss": 0.0806,
      "num_tokens": 3347804.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.23928211070597172,
      "rewards/accuracy_reward": 0.0989583320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.5729675292969,
      "epoch": 0.007681228996639462,
      "grad_norm": 0.2353515625,
      "learning_rate": 7.662835249042144e-08,
      "loss": 0.1188,
      "num_tokens": 3530027.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.21688513830304146,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.0834045410156,
      "epoch": 0.008065290446471435,
      "grad_norm": 0.22265625,
      "learning_rate": 8.045977011494252e-08,
      "loss": 0.0642,
      "num_tokens": 3721227.0,
      "reward": 0.3645833432674408,
      "reward_std": 0.14210023917257786,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3437500074505806,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.9062805175781,
      "epoch": 0.008449351896303409,
      "grad_norm": 0.2216796875,
      "learning_rate": 8.42911877394636e-08,
      "loss": 0.119,
      "num_tokens": 3883730.0,
      "reward": 0.5182291939854622,
      "reward_std": 0.2171967774629593,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140624925494194,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.4792175292969,
      "epoch": 0.00883341334613538,
      "grad_norm": 0.154296875,
      "learning_rate": 8.812260536398467e-08,
      "loss": 0.083,
      "num_tokens": 4079920.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.23546872846782207,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.7604370117188,
      "epoch": 0.009217474795967354,
      "grad_norm": 0.19140625,
      "learning_rate": 9.195402298850574e-08,
      "loss": 0.1199,
      "num_tokens": 4259505.0,
      "reward": 0.494791679084301,
      "reward_std": 0.24770202860236168,
      "rewards/accuracy_reward": 0.14583333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.1042175292969,
      "epoch": 0.009601536245799328,
      "grad_norm": 0.212890625,
      "learning_rate": 9.578544061302682e-08,
      "loss": 0.0638,
      "num_tokens": 4450227.0,
      "reward": 0.3750000074505806,
      "reward_std": 0.13964218832552433,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1478.8958435058594,
      "epoch": 0.009985597695631301,
      "grad_norm": 0.244140625,
      "learning_rate": 9.961685823754789e-08,
      "loss": 0.0566,
      "num_tokens": 4605169.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.21222718060016632,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.0000610351562,
      "epoch": 0.010369659145463275,
      "grad_norm": 0.2138671875,
      "learning_rate": 1.0344827586206897e-07,
      "loss": 0.0572,
      "num_tokens": 4771825.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.1600092607550323,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1948.4271240234375,
      "epoch": 0.010753720595295247,
      "grad_norm": 0.1611328125,
      "learning_rate": 1.0727969348659003e-07,
      "loss": 0.0553,
      "num_tokens": 4974210.0,
      "reward": 0.3125000149011612,
      "reward_std": 0.06864439975470304,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.8646545410156,
      "epoch": 0.01113778204512722,
      "grad_norm": 0.1962890625,
      "learning_rate": 1.111111111111111e-07,
      "loss": 0.0738,
      "num_tokens": 5141749.0,
      "reward": 0.5000000223517418,
      "reward_std": 0.1964168194681406,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.7604675292969,
      "epoch": 0.011521843494959194,
      "grad_norm": 0.1767578125,
      "learning_rate": 1.1494252873563217e-07,
      "loss": 0.0871,
      "num_tokens": 5319406.0,
      "reward": 0.4817708432674408,
      "reward_std": 0.15097743971273303,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.8334045410156,
      "epoch": 0.011905904944791167,
      "grad_norm": 0.19921875,
      "learning_rate": 1.1877394636015325e-07,
      "loss": 0.0786,
      "num_tokens": 5497374.0,
      "reward": 0.4244791865348816,
      "reward_std": 0.14389221463352442,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.6771240234375,
      "epoch": 0.012289966394623139,
      "grad_norm": 0.220703125,
      "learning_rate": 1.2260536398467432e-07,
      "loss": 0.0935,
      "num_tokens": 5675727.0,
      "reward": 0.4166666865348816,
      "reward_std": 0.19211183860898018,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.1562805175781,
      "epoch": 0.012674027844455113,
      "grad_norm": 0.2138671875,
      "learning_rate": 1.2643678160919542e-07,
      "loss": 0.0906,
      "num_tokens": 5845726.0,
      "reward": 0.5390625,
      "reward_std": 0.3064126968383789,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1506.4896240234375,
      "epoch": 0.013058089294287086,
      "grad_norm": 0.224609375,
      "learning_rate": 1.3026819923371645e-07,
      "loss": 0.1198,
      "num_tokens": 6008581.0,
      "reward": 0.5729166865348816,
      "reward_std": 0.2791721299290657,
      "rewards/accuracy_reward": 0.14583333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.2604370117188,
      "epoch": 0.01344215074411906,
      "grad_norm": 0.21484375,
      "learning_rate": 1.3409961685823753e-07,
      "loss": 0.0535,
      "num_tokens": 6196590.0,
      "reward": 0.3593750074505806,
      "reward_std": 0.10984884202480316,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.1979675292969,
      "epoch": 0.013826212193951032,
      "grad_norm": 0.2060546875,
      "learning_rate": 1.379310344827586e-07,
      "loss": 0.097,
      "num_tokens": 6359641.0,
      "reward": 0.5234375223517418,
      "reward_std": 0.22046434879302979,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875074505806,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1667.7708740234375,
      "epoch": 0.014210273643783005,
      "grad_norm": 0.177734375,
      "learning_rate": 1.4176245210727968e-07,
      "loss": 0.0698,
      "num_tokens": 6534163.0,
      "reward": 0.4531250074505806,
      "reward_std": 0.15216556936502457,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.1250610351562,
      "epoch": 0.014594335093614979,
      "grad_norm": 0.2109375,
      "learning_rate": 1.4559386973180076e-07,
      "loss": 0.0844,
      "num_tokens": 6703055.0,
      "reward": 0.505208358168602,
      "reward_std": 0.16614773403853178,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.442708320915699,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.0625305175781,
      "epoch": 0.014978396543446952,
      "grad_norm": 0.2119140625,
      "learning_rate": 1.4942528735632184e-07,
      "loss": 0.0985,
      "num_tokens": 6872245.0,
      "reward": 0.5052083358168602,
      "reward_std": 0.222918301820755,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.9792175292969,
      "epoch": 0.015362457993278924,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.532567049808429e-07,
      "loss": 0.0885,
      "num_tokens": 7058723.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.15408538281917572,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.6042175292969,
      "epoch": 0.0157465194431109,
      "grad_norm": 0.2080078125,
      "learning_rate": 1.5708812260536397e-07,
      "loss": 0.1148,
      "num_tokens": 7215477.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.17729653744027019,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.9271240234375,
      "epoch": 0.01613058089294287,
      "grad_norm": 0.1962890625,
      "learning_rate": 1.6091954022988505e-07,
      "loss": 0.0818,
      "num_tokens": 7385582.0,
      "reward": 0.5026041716337204,
      "reward_std": 0.22154850885272026,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.9167175292969,
      "epoch": 0.016514642342774843,
      "grad_norm": 0.232421875,
      "learning_rate": 1.6475095785440612e-07,
      "loss": 0.0942,
      "num_tokens": 7541446.0,
      "reward": 0.5807291865348816,
      "reward_std": 0.22830085828900337,
      "rewards/accuracy_reward": 0.15625000558793545,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1648.4687805175781,
      "epoch": 0.016898703792606818,
      "grad_norm": 0.2353515625,
      "learning_rate": 1.685823754789272e-07,
      "loss": 0.148,
      "num_tokens": 7712835.0,
      "reward": 0.4505208358168602,
      "reward_std": 0.20120975375175476,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.2396240234375,
      "epoch": 0.01728276524243879,
      "grad_norm": 0.208984375,
      "learning_rate": 1.7241379310344828e-07,
      "loss": 0.1147,
      "num_tokens": 7886850.0,
      "reward": 0.4375,
      "reward_std": 0.16076627373695374,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1691.3229675292969,
      "epoch": 0.01766682669227076,
      "grad_norm": 0.2119140625,
      "learning_rate": 1.7624521072796933e-07,
      "loss": 0.0228,
      "num_tokens": 8064473.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.11321644019335508,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1391.3020935058594,
      "epoch": 0.018050888142102737,
      "grad_norm": 0.255859375,
      "learning_rate": 1.800766283524904e-07,
      "loss": 0.0972,
      "num_tokens": 8215030.0,
      "reward": 0.5442708507180214,
      "reward_std": 0.23935068398714066,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.166748046875,
      "epoch": 0.01843494959193471,
      "grad_norm": 0.1748046875,
      "learning_rate": 1.839080459770115e-07,
      "loss": 0.0828,
      "num_tokens": 8383070.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.1704807048663497,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.7292175292969,
      "epoch": 0.018819011041766684,
      "grad_norm": 0.1904296875,
      "learning_rate": 1.8773946360153257e-07,
      "loss": 0.086,
      "num_tokens": 8569620.0,
      "reward": 0.427083358168602,
      "reward_std": 0.13604175299406052,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666567325592,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.6562805175781,
      "epoch": 0.019203072491598656,
      "grad_norm": 0.2412109375,
      "learning_rate": 1.9157088122605365e-07,
      "loss": 0.1386,
      "num_tokens": 8738995.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.2550000734627247,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.8437805175781,
      "epoch": 0.019587133941430628,
      "grad_norm": 0.1923828125,
      "learning_rate": 1.9540229885057472e-07,
      "loss": 0.0546,
      "num_tokens": 8929052.0,
      "reward": 0.408854179084301,
      "reward_std": 0.15740164648741484,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.3854675292969,
      "epoch": 0.019971195391262603,
      "grad_norm": 0.181640625,
      "learning_rate": 1.9923371647509578e-07,
      "loss": 0.0948,
      "num_tokens": 9107209.0,
      "reward": 0.432291679084301,
      "reward_std": 0.18800302036106586,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.7604370117188,
      "epoch": 0.020355256841094575,
      "grad_norm": 0.2578125,
      "learning_rate": 2.0306513409961685e-07,
      "loss": 0.1119,
      "num_tokens": 9281514.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.1955499481409788,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.3021240234375,
      "epoch": 0.02073931829092655,
      "grad_norm": 0.1904296875,
      "learning_rate": 2.0689655172413793e-07,
      "loss": 0.0931,
      "num_tokens": 9451023.0,
      "reward": 0.377604179084301,
      "reward_std": 0.11223115772008896,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.0104370117188,
      "epoch": 0.02112337974075852,
      "grad_norm": 0.26171875,
      "learning_rate": 2.1072796934865898e-07,
      "loss": 0.1671,
      "num_tokens": 9619968.0,
      "reward": 0.424479179084301,
      "reward_std": 0.14174982346594334,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.7396240234375,
      "epoch": 0.021507441190590493,
      "grad_norm": 0.2060546875,
      "learning_rate": 2.1455938697318006e-07,
      "loss": 0.0628,
      "num_tokens": 9799047.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.19780022278428078,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.0521240234375,
      "epoch": 0.02189150264042247,
      "grad_norm": 0.2119140625,
      "learning_rate": 2.1839080459770114e-07,
      "loss": 0.0781,
      "num_tokens": 9975596.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.14912654645740986,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333432674408,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.6146240234375,
      "epoch": 0.02227556409025444,
      "grad_norm": 0.2197265625,
      "learning_rate": 2.222222222222222e-07,
      "loss": 0.1292,
      "num_tokens": 10146519.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.2756075784564018,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.0833435058594,
      "epoch": 0.022659625540086412,
      "grad_norm": 0.1953125,
      "learning_rate": 2.2605363984674327e-07,
      "loss": 0.0767,
      "num_tokens": 10320183.0,
      "reward": 0.526041679084301,
      "reward_std": 0.24159196764230728,
      "rewards/accuracy_reward": 0.13541666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.1771545410156,
      "epoch": 0.023043686989918388,
      "grad_norm": 0.2001953125,
      "learning_rate": 2.2988505747126435e-07,
      "loss": 0.0948,
      "num_tokens": 10499448.0,
      "reward": 0.4348958358168602,
      "reward_std": 0.11261442862451077,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.5312805175781,
      "epoch": 0.02342774843975036,
      "grad_norm": 0.19921875,
      "learning_rate": 2.3371647509578543e-07,
      "loss": 0.0952,
      "num_tokens": 10685123.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.14470268972218037,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1467.6458740234375,
      "epoch": 0.023811809889582335,
      "grad_norm": 0.28515625,
      "learning_rate": 2.375478927203065e-07,
      "loss": 0.0842,
      "num_tokens": 10843393.0,
      "reward": 0.552083358168602,
      "reward_std": 0.2098541595041752,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.3333740234375,
      "epoch": 0.024195871339414306,
      "grad_norm": 0.1669921875,
      "learning_rate": 2.413793103448276e-07,
      "loss": 0.0631,
      "num_tokens": 11031273.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.18328907620161772,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1503.8646240234375,
      "epoch": 0.024579932789246278,
      "grad_norm": 0.244140625,
      "learning_rate": 2.4521072796934863e-07,
      "loss": 0.0757,
      "num_tokens": 11192012.0,
      "reward": 0.5078125074505806,
      "reward_std": 0.22904044017195702,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1906.09375,
      "epoch": 0.024963994239078253,
      "grad_norm": 0.177734375,
      "learning_rate": 2.490421455938697e-07,
      "loss": 0.0762,
      "num_tokens": 11393061.0,
      "reward": 0.330729179084301,
      "reward_std": 0.1098504769615829,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291641831398,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1819.4063110351562,
      "epoch": 0.025348055688910225,
      "grad_norm": 0.205078125,
      "learning_rate": 2.5287356321839084e-07,
      "loss": 0.1046,
      "num_tokens": 11582716.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.1561384303495288,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.4583740234375,
      "epoch": 0.025732117138742197,
      "grad_norm": 0.263671875,
      "learning_rate": 2.5670498084291184e-07,
      "loss": 0.1235,
      "num_tokens": 11737792.0,
      "reward": 0.591145858168602,
      "reward_std": 0.31366991251707077,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.5,
      "epoch": 0.026116178588574172,
      "grad_norm": 0.1748046875,
      "learning_rate": 2.605363984674329e-07,
      "loss": 0.0599,
      "num_tokens": 11908624.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.16151710227131844,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.8541870117188,
      "epoch": 0.026500240038406144,
      "grad_norm": 0.2080078125,
      "learning_rate": 2.64367816091954e-07,
      "loss": 0.0827,
      "num_tokens": 12091090.0,
      "reward": 0.4401041865348816,
      "reward_std": 0.2139188265427947,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.8021240234375,
      "epoch": 0.02688430148823812,
      "grad_norm": 0.1953125,
      "learning_rate": 2.6819923371647505e-07,
      "loss": 0.0679,
      "num_tokens": 12269855.0,
      "reward": 0.3671875149011612,
      "reward_std": 0.10924470145255327,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671874925494194,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.1041870117188,
      "epoch": 0.02726836293807009,
      "grad_norm": 0.244140625,
      "learning_rate": 2.7203065134099616e-07,
      "loss": 0.1305,
      "num_tokens": 12450545.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.17434079898521304,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.3125610351562,
      "epoch": 0.027652424387902063,
      "grad_norm": 0.224609375,
      "learning_rate": 2.758620689655172e-07,
      "loss": 0.111,
      "num_tokens": 12631559.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.1935316026210785,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.7396240234375,
      "epoch": 0.02803648583773404,
      "grad_norm": 0.2431640625,
      "learning_rate": 2.7969348659003826e-07,
      "loss": 0.0786,
      "num_tokens": 12803542.0,
      "reward": 0.4114583507180214,
      "reward_std": 0.15428296942263842,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1722.7917175292969,
      "epoch": 0.02842054728756601,
      "grad_norm": 0.2021484375,
      "learning_rate": 2.8352490421455936e-07,
      "loss": 0.0945,
      "num_tokens": 12982818.0,
      "reward": 0.3854166865348816,
      "reward_std": 0.10337093006819487,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166567325592,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.5833740234375,
      "epoch": 0.028804608737397985,
      "grad_norm": 0.21875,
      "learning_rate": 2.873563218390804e-07,
      "loss": 0.0927,
      "num_tokens": 13151114.0,
      "reward": 0.393229179084301,
      "reward_std": 0.1209185216575861,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1948.6771240234375,
      "epoch": 0.029188670187229957,
      "grad_norm": 0.158203125,
      "learning_rate": 2.911877394636015e-07,
      "loss": 0.0618,
      "num_tokens": 13351947.0,
      "reward": 0.338541679084301,
      "reward_std": 0.14428583858534694,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3281249925494194,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.9270935058594,
      "epoch": 0.02957273163706193,
      "grad_norm": 0.1376953125,
      "learning_rate": 2.9501915708812257e-07,
      "loss": 0.0583,
      "num_tokens": 13536972.0,
      "reward": 0.408854179084301,
      "reward_std": 0.1568221254274249,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.9687805175781,
      "epoch": 0.029956793086893904,
      "grad_norm": 0.205078125,
      "learning_rate": 2.988505747126437e-07,
      "loss": 0.0138,
      "num_tokens": 13719601.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.15776514215394855,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1515.3750610351562,
      "epoch": 0.030340854536725876,
      "grad_norm": 0.2080078125,
      "learning_rate": 3.0268199233716473e-07,
      "loss": 0.0687,
      "num_tokens": 13877357.0,
      "reward": 0.5755208432674408,
      "reward_std": 0.25233107432723045,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.9687805175781,
      "epoch": 0.030724915986557848,
      "grad_norm": 0.2041015625,
      "learning_rate": 3.065134099616858e-07,
      "loss": 0.0706,
      "num_tokens": 14039562.0,
      "reward": 0.4739583507180214,
      "reward_std": 0.1672044163569808,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.2187805175781,
      "epoch": 0.031108977436389823,
      "grad_norm": 0.205078125,
      "learning_rate": 3.103448275862069e-07,
      "loss": 0.1482,
      "num_tokens": 14223063.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.18344663083553314,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3437500074505806,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.4791870117188,
      "epoch": 0.0314930388862218,
      "grad_norm": 0.19921875,
      "learning_rate": 3.1417624521072794e-07,
      "loss": 0.1069,
      "num_tokens": 14405493.0,
      "reward": 0.4557291716337204,
      "reward_std": 0.18137156032025814,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.9896240234375,
      "epoch": 0.03187710033605377,
      "grad_norm": 0.1923828125,
      "learning_rate": 3.1800766283524904e-07,
      "loss": 0.0674,
      "num_tokens": 14581660.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.153211138676852,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3750000074505806,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1448.5521240234375,
      "epoch": 0.03226116178588574,
      "grad_norm": 0.248046875,
      "learning_rate": 3.218390804597701e-07,
      "loss": 0.0761,
      "num_tokens": 14733665.0,
      "reward": 0.5364583432674408,
      "reward_std": 0.21830646321177483,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.7604370117188,
      "epoch": 0.03264522323571772,
      "grad_norm": 0.1953125,
      "learning_rate": 3.2567049808429114e-07,
      "loss": 0.0567,
      "num_tokens": 14908242.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.28278843872249126,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1754.9687805175781,
      "epoch": 0.033029284685549685,
      "grad_norm": 0.2275390625,
      "learning_rate": 3.2950191570881225e-07,
      "loss": 0.0805,
      "num_tokens": 15091831.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.2159718368202448,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1491.8750610351562,
      "epoch": 0.03341334613538166,
      "grad_norm": 0.2255859375,
      "learning_rate": 3.333333333333333e-07,
      "loss": 0.0923,
      "num_tokens": 15249755.0,
      "reward": 0.4661458507180214,
      "reward_std": 0.17951179295778275,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1938.3437805175781,
      "epoch": 0.033797407585213636,
      "grad_norm": 0.1591796875,
      "learning_rate": 3.371647509578544e-07,
      "loss": 0.0476,
      "num_tokens": 15449412.0,
      "reward": 0.385416679084301,
      "reward_std": 0.22161301178857684,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3020833283662796,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.1979370117188,
      "epoch": 0.034181469035045604,
      "grad_norm": 0.205078125,
      "learning_rate": 3.4099616858237546e-07,
      "loss": 0.0643,
      "num_tokens": 15619479.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.2524689231067896,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.0625610351562,
      "epoch": 0.03456553048487758,
      "grad_norm": 0.26953125,
      "learning_rate": 3.4482758620689656e-07,
      "loss": 0.1039,
      "num_tokens": 15798805.0,
      "reward": 0.424479179084301,
      "reward_std": 0.18616511300206184,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.1875305175781,
      "epoch": 0.034949591934709555,
      "grad_norm": 0.1943359375,
      "learning_rate": 3.486590038314176e-07,
      "loss": 0.0819,
      "num_tokens": 15968855.0,
      "reward": 0.622395858168602,
      "reward_std": 0.3582340069115162,
      "rewards/accuracy_reward": 0.18749999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.6250610351562,
      "epoch": 0.03533365338454152,
      "grad_norm": 0.19140625,
      "learning_rate": 3.5249042145593867e-07,
      "loss": 0.0621,
      "num_tokens": 16159715.0,
      "reward": 0.3906250074505806,
      "reward_std": 0.15856974385678768,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.5000610351562,
      "epoch": 0.0357177148343735,
      "grad_norm": 0.1943359375,
      "learning_rate": 3.5632183908045977e-07,
      "loss": 0.0571,
      "num_tokens": 16345275.0,
      "reward": 0.395833358168602,
      "reward_std": 0.15551274083554745,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3749999925494194,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1609.1458740234375,
      "epoch": 0.036101776284205474,
      "grad_norm": 0.2236328125,
      "learning_rate": 3.601532567049808e-07,
      "loss": 0.1229,
      "num_tokens": 16511369.0,
      "reward": 0.5937500074505806,
      "reward_std": 0.25633867271244526,
      "rewards/accuracy_reward": 0.18750000651925802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.1667175292969,
      "epoch": 0.03648583773403745,
      "grad_norm": 0.2001953125,
      "learning_rate": 3.639846743295019e-07,
      "loss": 0.1022,
      "num_tokens": 16672833.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.22069755382835865,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.1875305175781,
      "epoch": 0.03686989918386942,
      "grad_norm": 0.22265625,
      "learning_rate": 3.67816091954023e-07,
      "loss": 0.1914,
      "num_tokens": 16851379.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.17728853411972523,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3593749925494194,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.5417175292969,
      "epoch": 0.03725396063370139,
      "grad_norm": 0.212890625,
      "learning_rate": 3.716475095785441e-07,
      "loss": 0.0801,
      "num_tokens": 17014079.0,
      "reward": 0.5208333507180214,
      "reward_std": 0.24689505621790886,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.4583740234375,
      "epoch": 0.03763802208353337,
      "grad_norm": 0.1845703125,
      "learning_rate": 3.7547892720306513e-07,
      "loss": 0.0515,
      "num_tokens": 17194619.0,
      "reward": 0.4713541865348816,
      "reward_std": 0.20721785631030798,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.5729675292969,
      "epoch": 0.038022083533365336,
      "grad_norm": 0.2373046875,
      "learning_rate": 3.793103448275862e-07,
      "loss": 0.1497,
      "num_tokens": 17365802.0,
      "reward": 0.5468750074505806,
      "reward_std": 0.3015727996826172,
      "rewards/accuracy_reward": 0.16666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.0417175292969,
      "epoch": 0.03840614498319731,
      "grad_norm": 0.20703125,
      "learning_rate": 3.831417624521073e-07,
      "loss": 0.0888,
      "num_tokens": 17538886.0,
      "reward": 0.4791666716337204,
      "reward_std": 0.17502420209348202,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.5104675292969,
      "epoch": 0.03879020643302929,
      "grad_norm": 0.2001953125,
      "learning_rate": 3.8697318007662834e-07,
      "loss": 0.0621,
      "num_tokens": 17714671.0,
      "reward": 0.510416679084301,
      "reward_std": 0.20976222679018974,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1634.6562805175781,
      "epoch": 0.039174267882861255,
      "grad_norm": 0.177734375,
      "learning_rate": 3.9080459770114945e-07,
      "loss": 0.0823,
      "num_tokens": 17885070.0,
      "reward": 0.432291679084301,
      "reward_std": 0.13325785100460052,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.4896240234375,
      "epoch": 0.03955832933269323,
      "grad_norm": 0.2412109375,
      "learning_rate": 3.946360153256705e-07,
      "loss": 0.1523,
      "num_tokens": 18061533.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.1777381021529436,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1327.0208587646484,
      "epoch": 0.039942390782525206,
      "grad_norm": 0.255859375,
      "learning_rate": 3.9846743295019155e-07,
      "loss": 0.1257,
      "num_tokens": 18206023.0,
      "reward": 0.645833358168602,
      "reward_std": 0.28297435492277145,
      "rewards/accuracy_reward": 0.1875000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333432674408,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.8854675292969,
      "epoch": 0.040326452232357174,
      "grad_norm": 0.2080078125,
      "learning_rate": 4.0229885057471266e-07,
      "loss": 0.0736,
      "num_tokens": 18386540.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.23162890691310167,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1860.0833740234375,
      "epoch": 0.04071051368218915,
      "grad_norm": 0.169921875,
      "learning_rate": 4.061302681992337e-07,
      "loss": 0.0585,
      "num_tokens": 18580036.0,
      "reward": 0.3906250223517418,
      "reward_std": 0.16623062966391444,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.0417175292969,
      "epoch": 0.041094575132021124,
      "grad_norm": 0.1708984375,
      "learning_rate": 4.099616858237548e-07,
      "loss": 0.0507,
      "num_tokens": 18749368.0,
      "reward": 0.463541679084301,
      "reward_std": 0.13008620403707027,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.1562805175781,
      "epoch": 0.0414786365818531,
      "grad_norm": 0.2197265625,
      "learning_rate": 4.1379310344827586e-07,
      "loss": 0.0764,
      "num_tokens": 18933951.0,
      "reward": 0.4270833358168602,
      "reward_std": 0.21493767574429512,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1914.5208740234375,
      "epoch": 0.04186269803168507,
      "grad_norm": 0.13671875,
      "learning_rate": 4.1762452107279697e-07,
      "loss": 0.0302,
      "num_tokens": 19133401.0,
      "reward": 0.3750000149011612,
      "reward_std": 0.193430557847023,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1397.6250305175781,
      "epoch": 0.04224675948151704,
      "grad_norm": 0.265625,
      "learning_rate": 4.2145593869731797e-07,
      "loss": 0.0547,
      "num_tokens": 19282189.0,
      "reward": 0.59375,
      "reward_std": 0.2414940595626831,
      "rewards/accuracy_reward": 0.14583333022892475,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.9271545410156,
      "epoch": 0.04263082093134902,
      "grad_norm": 0.2216796875,
      "learning_rate": 4.25287356321839e-07,
      "loss": 0.031,
      "num_tokens": 19454374.0,
      "reward": 0.643229179084301,
      "reward_std": 0.20937169808894396,
      "rewards/accuracy_reward": 0.2500000074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.8646240234375,
      "epoch": 0.04301488238118099,
      "grad_norm": 0.224609375,
      "learning_rate": 4.291187739463601e-07,
      "loss": 0.0995,
      "num_tokens": 19632017.0,
      "reward": 0.4192708507180214,
      "reward_std": 0.1467351485043764,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.0729675292969,
      "epoch": 0.04339894383101296,
      "grad_norm": 0.2080078125,
      "learning_rate": 4.329501915708812e-07,
      "loss": 0.0525,
      "num_tokens": 19812800.0,
      "reward": 0.3906250074505806,
      "reward_std": 0.15219017304480076,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.4063110351562,
      "epoch": 0.04378300528084494,
      "grad_norm": 0.19140625,
      "learning_rate": 4.367816091954023e-07,
      "loss": 0.1018,
      "num_tokens": 19997175.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.2861969545483589,
      "rewards/accuracy_reward": 0.12500000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1806.3854370117188,
      "epoch": 0.044167066730676906,
      "grad_norm": 0.1953125,
      "learning_rate": 4.4061302681992333e-07,
      "loss": 0.0735,
      "num_tokens": 20188180.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.17212526127696037,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.5208587646484,
      "epoch": 0.04455112818050888,
      "grad_norm": 0.2138671875,
      "learning_rate": 4.444444444444444e-07,
      "loss": 0.0587,
      "num_tokens": 20351502.0,
      "reward": 0.6041666939854622,
      "reward_std": 0.2561345724388957,
      "rewards/accuracy_reward": 0.1979166716337204,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.6562805175781,
      "epoch": 0.044935189630340856,
      "grad_norm": 0.1640625,
      "learning_rate": 4.482758620689655e-07,
      "loss": 0.0553,
      "num_tokens": 20530589.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.17080461606383324,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.7188110351562,
      "epoch": 0.045319251080172825,
      "grad_norm": 0.197265625,
      "learning_rate": 4.5210727969348654e-07,
      "loss": 0.0524,
      "num_tokens": 20709890.0,
      "reward": 0.5104166865348816,
      "reward_std": 0.18949182983487844,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.4375610351562,
      "epoch": 0.0457033125300048,
      "grad_norm": 0.1708984375,
      "learning_rate": 4.5593869731800764e-07,
      "loss": 0.0606,
      "num_tokens": 20889668.0,
      "reward": 0.424479179084301,
      "reward_std": 0.20285394228994846,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791567325592,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1420.7708740234375,
      "epoch": 0.046087373979836775,
      "grad_norm": 0.224609375,
      "learning_rate": 4.597701149425287e-07,
      "loss": 0.0155,
      "num_tokens": 21039710.0,
      "reward": 0.6458333432674408,
      "reward_std": 0.28165198117494583,
      "rewards/accuracy_reward": 0.22916666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1846.2188110351562,
      "epoch": 0.04647143542966875,
      "grad_norm": 0.138671875,
      "learning_rate": 4.636015325670498e-07,
      "loss": 0.0235,
      "num_tokens": 21232083.0,
      "reward": 0.369791679084301,
      "reward_std": 0.1359492796473205,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416641831398,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.7500610351562,
      "epoch": 0.04685549687950072,
      "grad_norm": 0.2080078125,
      "learning_rate": 4.6743295019157085e-07,
      "loss": 0.0817,
      "num_tokens": 21418923.0,
      "reward": 0.5000000223517418,
      "reward_std": 0.22314649540930986,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1531.2916870117188,
      "epoch": 0.047239558329332694,
      "grad_norm": 0.2216796875,
      "learning_rate": 4.712643678160919e-07,
      "loss": 0.0584,
      "num_tokens": 21580583.0,
      "reward": 0.5104166865348816,
      "reward_std": 0.19526461325585842,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166567325592,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.1250305175781,
      "epoch": 0.04762361977916467,
      "grad_norm": 0.171875,
      "learning_rate": 4.75095785440613e-07,
      "loss": 0.0856,
      "num_tokens": 21750803.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.13608664087951183,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.5312805175781,
      "epoch": 0.04800768122899664,
      "grad_norm": 0.2470703125,
      "learning_rate": 4.789272030651341e-07,
      "loss": 0.0997,
      "num_tokens": 21925726.0,
      "reward": 0.3854166716337204,
      "reward_std": 0.12451678980141878,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1879.2083740234375,
      "epoch": 0.04839174267882861,
      "grad_norm": 0.1982421875,
      "learning_rate": 4.827586206896552e-07,
      "loss": 0.0667,
      "num_tokens": 22121162.0,
      "reward": 0.450520858168602,
      "reward_std": 0.17516027949750423,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541641831398,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1839.1146545410156,
      "epoch": 0.04877580412866059,
      "grad_norm": 0.1396484375,
      "learning_rate": 4.865900383141763e-07,
      "loss": 0.0649,
      "num_tokens": 22312741.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.20003637205809355,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333283662796,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1740.4895935058594,
      "epoch": 0.049159865578492556,
      "grad_norm": 0.1611328125,
      "learning_rate": 4.904214559386973e-07,
      "loss": 0.0975,
      "num_tokens": 22493068.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.17907519824802876,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.5937805175781,
      "epoch": 0.04954392702832453,
      "grad_norm": 0.2177734375,
      "learning_rate": 4.942528735632184e-07,
      "loss": 0.1182,
      "num_tokens": 22654765.0,
      "reward": 0.5416666865348816,
      "reward_std": 0.22721511498093605,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.1979675292969,
      "epoch": 0.04992798847815651,
      "grad_norm": 0.2216796875,
      "learning_rate": 4.980842911877394e-07,
      "loss": 0.1092,
      "num_tokens": 22810976.0,
      "reward": 0.450520858168602,
      "reward_std": 0.12435647565871477,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419270820915699,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1305.2396545410156,
      "epoch": 0.050312049927988475,
      "grad_norm": 0.267578125,
      "learning_rate": 5.019157088122606e-07,
      "loss": 0.093,
      "num_tokens": 22949751.0,
      "reward": 0.6614583432674408,
      "reward_std": 0.26423680409789085,
      "rewards/accuracy_reward": 0.21875000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.6875305175781,
      "epoch": 0.05069611137782045,
      "grad_norm": 0.20703125,
      "learning_rate": 5.057471264367817e-07,
      "loss": 0.0875,
      "num_tokens": 23127993.0,
      "reward": 0.502604179084301,
      "reward_std": 0.24153999611735344,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.0313110351562,
      "epoch": 0.051080172827652426,
      "grad_norm": 0.193359375,
      "learning_rate": 5.095785440613026e-07,
      "loss": 0.069,
      "num_tokens": 23305972.0,
      "reward": 0.4635416716337204,
      "reward_std": 0.17044265195727348,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1826.0416870117188,
      "epoch": 0.051464234277484394,
      "grad_norm": 0.173828125,
      "learning_rate": 5.134099616858237e-07,
      "loss": 0.0644,
      "num_tokens": 23498360.0,
      "reward": 0.3828125074505806,
      "reward_std": 0.15769892698153853,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1922.8020935058594,
      "epoch": 0.05184829572731637,
      "grad_norm": 0.1787109375,
      "learning_rate": 5.172413793103448e-07,
      "loss": 0.0579,
      "num_tokens": 23697061.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.2061835192143917,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.328125,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.0834045410156,
      "epoch": 0.052232357177148345,
      "grad_norm": 0.265625,
      "learning_rate": 5.210727969348658e-07,
      "loss": 0.0844,
      "num_tokens": 23858133.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.26548151299357414,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1940.0104370117188,
      "epoch": 0.05261641862698032,
      "grad_norm": 0.1611328125,
      "learning_rate": 5.249042145593869e-07,
      "loss": 0.0918,
      "num_tokens": 24058462.0,
      "reward": 0.2968750074505806,
      "reward_std": 0.08337579295039177,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.296875,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1679.5312805175781,
      "epoch": 0.05300048007681229,
      "grad_norm": 0.203125,
      "learning_rate": 5.28735632183908e-07,
      "loss": 0.0797,
      "num_tokens": 24236233.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.1902182660996914,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.4896240234375,
      "epoch": 0.053384541526644264,
      "grad_norm": 0.1923828125,
      "learning_rate": 5.325670498084291e-07,
      "loss": 0.0962,
      "num_tokens": 24420680.0,
      "reward": 0.4635416865348816,
      "reward_std": 0.2791525460779667,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1452.3958435058594,
      "epoch": 0.05376860297647624,
      "grad_norm": 0.2001953125,
      "learning_rate": 5.363984674329501e-07,
      "loss": 0.0602,
      "num_tokens": 24574958.0,
      "reward": 0.4583333507180214,
      "reward_std": 0.13799087703227997,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166567325592,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1835.0104370117188,
      "epoch": 0.05415266442630821,
      "grad_norm": 0.193359375,
      "learning_rate": 5.402298850574712e-07,
      "loss": 0.0863,
      "num_tokens": 24765087.0,
      "reward": 0.3619791716337204,
      "reward_std": 0.134540980681777,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1642.3438110351562,
      "epoch": 0.05453672587614018,
      "grad_norm": 0.251953125,
      "learning_rate": 5.440613026819923e-07,
      "loss": 0.0833,
      "num_tokens": 24935096.0,
      "reward": 0.4192708507180214,
      "reward_std": 0.16582264378666878,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.0625,
      "epoch": 0.05492078732597216,
      "grad_norm": 0.1806640625,
      "learning_rate": 5.478927203065133e-07,
      "loss": 0.1043,
      "num_tokens": 25119782.0,
      "reward": 0.369791679084301,
      "reward_std": 0.12501750700175762,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1656.1562805175781,
      "epoch": 0.055304848775804126,
      "grad_norm": 0.173828125,
      "learning_rate": 5.517241379310344e-07,
      "loss": 0.0502,
      "num_tokens": 25294565.0,
      "reward": 0.4583333507180214,
      "reward_std": 0.17004472948610783,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.5209045410156,
      "epoch": 0.0556889102256361,
      "grad_norm": 0.2080078125,
      "learning_rate": 5.555555555555555e-07,
      "loss": 0.1149,
      "num_tokens": 25463951.0,
      "reward": 0.533854179084301,
      "reward_std": 0.2579926773905754,
      "rewards/accuracy_reward": 0.14583333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.9062805175781,
      "epoch": 0.05607297167546808,
      "grad_norm": 0.19921875,
      "learning_rate": 5.593869731800765e-07,
      "loss": 0.1201,
      "num_tokens": 25631646.0,
      "reward": 0.5468750074505806,
      "reward_std": 0.28703723102808,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.1666870117188,
      "epoch": 0.056457033125300045,
      "grad_norm": 0.1904296875,
      "learning_rate": 5.632183908045976e-07,
      "loss": 0.0575,
      "num_tokens": 25804782.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.19034797605127096,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.5833740234375,
      "epoch": 0.05684109457513202,
      "grad_norm": 0.1826171875,
      "learning_rate": 5.670498084291187e-07,
      "loss": 0.0884,
      "num_tokens": 25994814.0,
      "reward": 0.4010416716337204,
      "reward_std": 0.14690229669213295,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.0104675292969,
      "epoch": 0.057225156024963995,
      "grad_norm": 0.1923828125,
      "learning_rate": 5.708812260536398e-07,
      "loss": 0.118,
      "num_tokens": 26185623.0,
      "reward": 0.3906250223517418,
      "reward_std": 0.1799048911780119,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416641831398,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1678.7500305175781,
      "epoch": 0.05760921747479597,
      "grad_norm": 0.185546875,
      "learning_rate": 5.747126436781608e-07,
      "loss": 0.0814,
      "num_tokens": 26362783.0,
      "reward": 0.4140625223517418,
      "reward_std": 0.17123159859329462,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791567325592,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1699.8229370117188,
      "epoch": 0.05799327892462794,
      "grad_norm": 0.1953125,
      "learning_rate": 5.785440613026819e-07,
      "loss": 0.1292,
      "num_tokens": 26539894.0,
      "reward": 0.408854179084301,
      "reward_std": 0.15536396764218807,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1843.3958740234375,
      "epoch": 0.058377340374459914,
      "grad_norm": 0.1640625,
      "learning_rate": 5.82375478927203e-07,
      "loss": 0.0511,
      "num_tokens": 26730452.0,
      "reward": 0.447916679084301,
      "reward_std": 0.1471330402418971,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.2500305175781,
      "epoch": 0.05876140182429189,
      "grad_norm": 0.2216796875,
      "learning_rate": 5.86206896551724e-07,
      "loss": 0.1127,
      "num_tokens": 26903140.0,
      "reward": 0.6354166865348816,
      "reward_std": 0.34830770641565323,
      "rewards/accuracy_reward": 0.22916667070239782,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.4479370117188,
      "epoch": 0.05914546327412386,
      "grad_norm": 0.1416015625,
      "learning_rate": 5.900383141762451e-07,
      "loss": -0.0104,
      "num_tokens": 27091359.0,
      "reward": 0.3020833358168602,
      "reward_std": 0.06181123945862055,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3020833358168602,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.135498046875,
      "epoch": 0.05952952472395583,
      "grad_norm": 0.2275390625,
      "learning_rate": 5.938697318007662e-07,
      "loss": 0.0621,
      "num_tokens": 27249828.0,
      "reward": 0.5598958432674408,
      "reward_std": 0.31126198545098305,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.9479370117188,
      "epoch": 0.05991358617378781,
      "grad_norm": 0.275390625,
      "learning_rate": 5.977011494252874e-07,
      "loss": 0.1725,
      "num_tokens": 27405847.0,
      "reward": 0.6328125298023224,
      "reward_std": 0.30618276819586754,
      "rewards/accuracy_reward": 0.19791666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958432674408,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.6875305175781,
      "epoch": 0.06029764762361978,
      "grad_norm": 0.2314453125,
      "learning_rate": 6.015325670498084e-07,
      "loss": 0.0264,
      "num_tokens": 27587881.0,
      "reward": 0.395833358168602,
      "reward_std": 0.12702175183221698,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166567325592,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.9688110351562,
      "epoch": 0.06068170907345175,
      "grad_norm": 0.2021484375,
      "learning_rate": 6.053639846743295e-07,
      "loss": 0.1195,
      "num_tokens": 27774750.0,
      "reward": 0.408854179084301,
      "reward_std": 0.15365522168576717,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.9896240234375,
      "epoch": 0.06106577052328373,
      "grad_norm": 0.1826171875,
      "learning_rate": 6.091954022988506e-07,
      "loss": 0.1137,
      "num_tokens": 27955781.0,
      "reward": 0.4687500074505806,
      "reward_std": 0.23078224807977676,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.4271240234375,
      "epoch": 0.061449831973115696,
      "grad_norm": 0.2412109375,
      "learning_rate": 6.130268199233716e-07,
      "loss": 0.0745,
      "num_tokens": 28124750.0,
      "reward": 0.4817708358168602,
      "reward_std": 0.2521183304488659,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.6771240234375,
      "epoch": 0.06183389342294767,
      "grad_norm": 0.1904296875,
      "learning_rate": 6.168582375478927e-07,
      "loss": 0.0906,
      "num_tokens": 28290943.0,
      "reward": 0.526041679084301,
      "reward_std": 0.21908506378531456,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.9167175292969,
      "epoch": 0.062217954872779646,
      "grad_norm": 0.185546875,
      "learning_rate": 6.206896551724138e-07,
      "loss": 0.0853,
      "num_tokens": 28470727.0,
      "reward": 0.5364583432674408,
      "reward_std": 0.18870128691196442,
      "rewards/accuracy_reward": 0.16666666232049465,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1579.2604675292969,
      "epoch": 0.06260201632261161,
      "grad_norm": 0.19921875,
      "learning_rate": 6.245210727969349e-07,
      "loss": 0.0556,
      "num_tokens": 28634544.0,
      "reward": 0.479166679084301,
      "reward_std": 0.18134189024567604,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.4271240234375,
      "epoch": 0.0629860777724436,
      "grad_norm": 0.18359375,
      "learning_rate": 6.283524904214559e-07,
      "loss": 0.0819,
      "num_tokens": 28807465.0,
      "reward": 0.408854179084301,
      "reward_std": 0.14228766225278378,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.0417175292969,
      "epoch": 0.06337013922227556,
      "grad_norm": 0.1630859375,
      "learning_rate": 6.32183908045977e-07,
      "loss": 0.0246,
      "num_tokens": 28982693.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.11226824019104242,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.697998046875,
      "epoch": 0.06375420067210753,
      "grad_norm": 0.244140625,
      "learning_rate": 6.360153256704981e-07,
      "loss": 0.0904,
      "num_tokens": 29152424.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.28810375928878784,
      "rewards/accuracy_reward": 0.15624999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.8021240234375,
      "epoch": 0.06413826212193952,
      "grad_norm": 0.185546875,
      "learning_rate": 6.398467432950191e-07,
      "loss": 0.0469,
      "num_tokens": 29326813.0,
      "reward": 0.5989583432674408,
      "reward_std": 0.2089410088956356,
      "rewards/accuracy_reward": 0.19791666232049465,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.5729675292969,
      "epoch": 0.06452232357177148,
      "grad_norm": 0.228515625,
      "learning_rate": 6.436781609195402e-07,
      "loss": 0.0932,
      "num_tokens": 29493812.0,
      "reward": 0.502604179084301,
      "reward_std": 0.20399035140872002,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.3229370117188,
      "epoch": 0.06490638502160345,
      "grad_norm": 0.1865234375,
      "learning_rate": 6.475095785440613e-07,
      "loss": 0.0161,
      "num_tokens": 29668675.0,
      "reward": 0.5572916939854622,
      "reward_std": 0.2530537284910679,
      "rewards/accuracy_reward": 0.16666666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.6666870117188,
      "epoch": 0.06529044647143543,
      "grad_norm": 0.1953125,
      "learning_rate": 6.513409961685823e-07,
      "loss": 0.1066,
      "num_tokens": 29836499.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.22895518504083157,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.5625305175781,
      "epoch": 0.0656745079212674,
      "grad_norm": 0.1748046875,
      "learning_rate": 6.551724137931034e-07,
      "loss": 0.111,
      "num_tokens": 30011761.0,
      "reward": 0.4739583507180214,
      "reward_std": 0.21206179820001125,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.6770935058594,
      "epoch": 0.06605856937109937,
      "grad_norm": 0.197265625,
      "learning_rate": 6.590038314176245e-07,
      "loss": 0.0957,
      "num_tokens": 30207818.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.1679173968732357,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.0729370117188,
      "epoch": 0.06644263082093135,
      "grad_norm": 0.203125,
      "learning_rate": 6.628352490421456e-07,
      "loss": 0.1336,
      "num_tokens": 30387865.0,
      "reward": 0.3671875074505806,
      "reward_std": 0.11911331675946712,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541641831398,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1863.3646240234375,
      "epoch": 0.06682669227076332,
      "grad_norm": 0.2353515625,
      "learning_rate": 6.666666666666666e-07,
      "loss": 0.1019,
      "num_tokens": 30581468.0,
      "reward": 0.3281250074505806,
      "reward_std": 0.10617846250534058,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3177083283662796,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.3229675292969,
      "epoch": 0.06721075372059529,
      "grad_norm": 0.1943359375,
      "learning_rate": 6.704980842911877e-07,
      "loss": 0.1027,
      "num_tokens": 30755987.0,
      "reward": 0.4401041716337204,
      "reward_std": 0.15784532111138105,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208432674408,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.3750610351562,
      "epoch": 0.06759481517042727,
      "grad_norm": 0.169921875,
      "learning_rate": 6.743295019157088e-07,
      "loss": 0.0889,
      "num_tokens": 30931295.0,
      "reward": 0.463541679084301,
      "reward_std": 0.1959435697644949,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1432.3541870117188,
      "epoch": 0.06797887662025924,
      "grad_norm": 0.21875,
      "learning_rate": 6.781609195402298e-07,
      "loss": 0.0924,
      "num_tokens": 31084849.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.1328337136656046,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1761.2604675292969,
      "epoch": 0.06836293807009121,
      "grad_norm": 0.1611328125,
      "learning_rate": 6.819923371647509e-07,
      "loss": 0.1065,
      "num_tokens": 31270618.0,
      "reward": 0.3437500074505806,
      "reward_std": 0.0831462568603456,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.34375,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.4896240234375,
      "epoch": 0.06874699951992319,
      "grad_norm": 0.1484375,
      "learning_rate": 6.85823754789272e-07,
      "loss": 0.0764,
      "num_tokens": 31460561.0,
      "reward": 0.3567708432674408,
      "reward_std": 0.12102555064484477,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3151041641831398,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.0208740234375,
      "epoch": 0.06913106096975516,
      "grad_norm": 0.2021484375,
      "learning_rate": 6.896551724137931e-07,
      "loss": 0.0963,
      "num_tokens": 31641699.0,
      "reward": 0.4765625223517418,
      "reward_std": 0.20853770896792412,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.1458740234375,
      "epoch": 0.06951512241958713,
      "grad_norm": 0.220703125,
      "learning_rate": 6.934865900383141e-07,
      "loss": 0.1189,
      "num_tokens": 31823337.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.285110954195261,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.5208740234375,
      "epoch": 0.06989918386941911,
      "grad_norm": 0.1865234375,
      "learning_rate": 6.973180076628352e-07,
      "loss": 0.037,
      "num_tokens": 32003963.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.14205067558214068,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.3646240234375,
      "epoch": 0.07028324531925108,
      "grad_norm": 0.1962890625,
      "learning_rate": 7.011494252873563e-07,
      "loss": 0.1199,
      "num_tokens": 32187270.0,
      "reward": 0.4088541865348816,
      "reward_std": 0.12591049168258905,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1878.6667175292969,
      "epoch": 0.07066730676908305,
      "grad_norm": 0.17578125,
      "learning_rate": 7.049808429118773e-07,
      "loss": 0.0474,
      "num_tokens": 32381662.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.20399256143718958,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.7292175292969,
      "epoch": 0.07105136821891503,
      "grad_norm": 0.20703125,
      "learning_rate": 7.088122605363984e-07,
      "loss": 0.0948,
      "num_tokens": 32557716.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.18026594817638397,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.1458435058594,
      "epoch": 0.071435429668747,
      "grad_norm": 0.2021484375,
      "learning_rate": 7.126436781609195e-07,
      "loss": 0.0483,
      "num_tokens": 32721802.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.1306017297320068,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.4479675292969,
      "epoch": 0.07181949111857898,
      "grad_norm": 0.1708984375,
      "learning_rate": 7.164750957854406e-07,
      "loss": 0.0617,
      "num_tokens": 32898453.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.15700350981205702,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1867.5416870117188,
      "epoch": 0.07220355256841095,
      "grad_norm": 0.1845703125,
      "learning_rate": 7.203065134099616e-07,
      "loss": 0.0511,
      "num_tokens": 33096049.0,
      "reward": 0.3854166865348816,
      "reward_std": 0.15223021619021893,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.34375,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.8437805175781,
      "epoch": 0.07258761401824292,
      "grad_norm": 0.2177734375,
      "learning_rate": 7.241379310344827e-07,
      "loss": 0.1412,
      "num_tokens": 33268626.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.17375362664461136,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.3333740234375,
      "epoch": 0.0729716754680749,
      "grad_norm": 0.19140625,
      "learning_rate": 7.279693486590039e-07,
      "loss": 0.0683,
      "num_tokens": 33445306.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.1827012598514557,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.8229370117188,
      "epoch": 0.07335573691790687,
      "grad_norm": 0.1875,
      "learning_rate": 7.318007662835249e-07,
      "loss": 0.047,
      "num_tokens": 33626633.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.1767996810376644,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.0833740234375,
      "epoch": 0.07373979836773883,
      "grad_norm": 0.2060546875,
      "learning_rate": 7.35632183908046e-07,
      "loss": 0.1014,
      "num_tokens": 33796369.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.23566409200429916,
      "rewards/accuracy_reward": 0.08854166604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.4584045410156,
      "epoch": 0.07412385981757082,
      "grad_norm": 0.19140625,
      "learning_rate": 7.394636015325671e-07,
      "loss": 0.1025,
      "num_tokens": 33973773.0,
      "reward": 0.4739583507180214,
      "reward_std": 0.17438989086076617,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1882.8646240234375,
      "epoch": 0.07450792126740279,
      "grad_norm": 0.1376953125,
      "learning_rate": 7.432950191570882e-07,
      "loss": 0.0611,
      "num_tokens": 34169600.0,
      "reward": 0.346354179084301,
      "reward_std": 0.1269625499844551,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3255208358168602,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1558.4687805175781,
      "epoch": 0.07489198271723475,
      "grad_norm": 0.20703125,
      "learning_rate": 7.471264367816092e-07,
      "loss": 0.0928,
      "num_tokens": 34332917.0,
      "reward": 0.463541679084301,
      "reward_std": 0.12047086097300053,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1634.6875305175781,
      "epoch": 0.07527604416706674,
      "grad_norm": 0.197265625,
      "learning_rate": 7.509578544061303e-07,
      "loss": 0.0779,
      "num_tokens": 34504887.0,
      "reward": 0.416666679084301,
      "reward_std": 0.12624988611787558,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.9583740234375,
      "epoch": 0.0756601056168987,
      "grad_norm": 0.20703125,
      "learning_rate": 7.547892720306514e-07,
      "loss": 0.0738,
      "num_tokens": 34688323.0,
      "reward": 0.463541679084301,
      "reward_std": 0.20355811528861523,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.2188110351562,
      "epoch": 0.07604416706673067,
      "grad_norm": 0.2177734375,
      "learning_rate": 7.586206896551724e-07,
      "loss": 0.098,
      "num_tokens": 34859320.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.15877362713217735,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1894.6667175292969,
      "epoch": 0.07642822851656265,
      "grad_norm": 0.1884765625,
      "learning_rate": 7.624521072796935e-07,
      "loss": 0.0996,
      "num_tokens": 35056848.0,
      "reward": 0.3750000149011612,
      "reward_std": 0.1475252229720354,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3229166716337204,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1863.3021240234375,
      "epoch": 0.07681228996639462,
      "grad_norm": 0.1748046875,
      "learning_rate": 7.662835249042146e-07,
      "loss": 0.0547,
      "num_tokens": 35252021.0,
      "reward": 0.3880208507180214,
      "reward_std": 0.19266861118376255,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541641831398,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.4062805175781,
      "epoch": 0.07719635141622659,
      "grad_norm": 0.232421875,
      "learning_rate": 7.701149425287356e-07,
      "loss": 0.1115,
      "num_tokens": 35417980.0,
      "reward": 0.5156250223517418,
      "reward_std": 0.22781999316066504,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1839.2292175292969,
      "epoch": 0.07758041286605857,
      "grad_norm": 0.1806640625,
      "learning_rate": 7.739463601532567e-07,
      "loss": 0.1197,
      "num_tokens": 35609010.0,
      "reward": 0.4140625149011612,
      "reward_std": 0.20704453997313976,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.15625,
      "epoch": 0.07796447431589054,
      "grad_norm": 0.24609375,
      "learning_rate": 7.777777777777778e-07,
      "loss": 0.0399,
      "num_tokens": 35761945.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.21726006641983986,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.2708740234375,
      "epoch": 0.07834853576572251,
      "grad_norm": 0.2080078125,
      "learning_rate": 7.816091954022989e-07,
      "loss": 0.082,
      "num_tokens": 35935395.0,
      "reward": 0.401041679084301,
      "reward_std": 0.1333829266950488,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1754.3021240234375,
      "epoch": 0.07873259721555449,
      "grad_norm": 0.15234375,
      "learning_rate": 7.854406130268199e-07,
      "loss": 0.103,
      "num_tokens": 36118760.0,
      "reward": 0.3359375074505806,
      "reward_std": 0.08789238287135959,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3255208358168602,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.3021240234375,
      "epoch": 0.07911665866538646,
      "grad_norm": 0.16796875,
      "learning_rate": 7.89272030651341e-07,
      "loss": 0.0928,
      "num_tokens": 36289405.0,
      "reward": 0.6328125149011612,
      "reward_std": 0.3209253214299679,
      "rewards/accuracy_reward": 0.19791666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.2395935058594,
      "epoch": 0.07950072011521843,
      "grad_norm": 0.228515625,
      "learning_rate": 7.931034482758621e-07,
      "loss": 0.1062,
      "num_tokens": 36464492.0,
      "reward": 0.432291679084301,
      "reward_std": 0.16767172142863274,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.9479675292969,
      "epoch": 0.07988478156505041,
      "grad_norm": 0.2158203125,
      "learning_rate": 7.969348659003831e-07,
      "loss": 0.0937,
      "num_tokens": 36642807.0,
      "reward": 0.4166666716337204,
      "reward_std": 0.16542065516114235,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3750000074505806,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.1771545410156,
      "epoch": 0.08026884301488238,
      "grad_norm": 0.2255859375,
      "learning_rate": 8.007662835249042e-07,
      "loss": 0.1203,
      "num_tokens": 36823480.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.17312548123300076,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.5104370117188,
      "epoch": 0.08065290446471435,
      "grad_norm": 0.142578125,
      "learning_rate": 8.045977011494253e-07,
      "loss": 0.0802,
      "num_tokens": 37010249.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.161942882463336,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.1458740234375,
      "epoch": 0.08103696591454633,
      "grad_norm": 0.2021484375,
      "learning_rate": 8.084291187739464e-07,
      "loss": 0.1221,
      "num_tokens": 37185071.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.18704628944396973,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.8958740234375,
      "epoch": 0.0814210273643783,
      "grad_norm": 0.15625,
      "learning_rate": 8.122605363984674e-07,
      "loss": 0.0605,
      "num_tokens": 37370293.0,
      "reward": 0.3802083432674408,
      "reward_std": 0.12084674090147018,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416641831398,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.3542175292969,
      "epoch": 0.08180508881421028,
      "grad_norm": 0.1845703125,
      "learning_rate": 8.160919540229885e-07,
      "loss": 0.0331,
      "num_tokens": 37552063.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.17819942720234394,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1691.3333435058594,
      "epoch": 0.08218915026404225,
      "grad_norm": 0.2080078125,
      "learning_rate": 8.199233716475096e-07,
      "loss": 0.0956,
      "num_tokens": 37730095.0,
      "reward": 0.385416679084301,
      "reward_std": 0.14018136821687222,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.0000610351562,
      "epoch": 0.08257321171387422,
      "grad_norm": 0.2451171875,
      "learning_rate": 8.237547892720306e-07,
      "loss": 0.113,
      "num_tokens": 37898487.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.1956240157596767,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1886.7187805175781,
      "epoch": 0.0829572731637062,
      "grad_norm": 0.1455078125,
      "learning_rate": 8.275862068965517e-07,
      "loss": 0.0419,
      "num_tokens": 38095124.0,
      "reward": 0.3177083432674408,
      "reward_std": 0.06348209269344807,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3177083358168602,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.5729675292969,
      "epoch": 0.08334133461353817,
      "grad_norm": 0.2099609375,
      "learning_rate": 8.314176245210728e-07,
      "loss": 0.1311,
      "num_tokens": 38266995.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.23105092719197273,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.260498046875,
      "epoch": 0.08372539606337014,
      "grad_norm": 0.205078125,
      "learning_rate": 8.352490421455939e-07,
      "loss": 0.0473,
      "num_tokens": 38439548.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.1933776643127203,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.5417175292969,
      "epoch": 0.08410945751320212,
      "grad_norm": 0.2119140625,
      "learning_rate": 8.390804597701148e-07,
      "loss": 0.0508,
      "num_tokens": 38612528.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.1605449616909027,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.9166870117188,
      "epoch": 0.08449351896303409,
      "grad_norm": 0.185546875,
      "learning_rate": 8.429118773946359e-07,
      "loss": 0.0956,
      "num_tokens": 38804000.0,
      "reward": 0.3984375074505806,
      "reward_std": 0.1844277773052454,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.9687805175781,
      "epoch": 0.08487758041286605,
      "grad_norm": 0.216796875,
      "learning_rate": 8.46743295019157e-07,
      "loss": 0.0912,
      "num_tokens": 38970565.0,
      "reward": 0.5598958507180214,
      "reward_std": 0.2687210235744715,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291641831398,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.7916870117188,
      "epoch": 0.08526164186269804,
      "grad_norm": 0.2177734375,
      "learning_rate": 8.50574712643678e-07,
      "loss": 0.1218,
      "num_tokens": 39154409.0,
      "reward": 0.4791666716337204,
      "reward_std": 0.22841137275099754,
      "rewards/accuracy_reward": 0.109375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.0312805175781,
      "epoch": 0.08564570331253,
      "grad_norm": 0.2255859375,
      "learning_rate": 8.544061302681991e-07,
      "loss": 0.0804,
      "num_tokens": 39332396.0,
      "reward": 0.513020858168602,
      "reward_std": 0.2054196409881115,
      "rewards/accuracy_reward": 0.14062499813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.9896240234375,
      "epoch": 0.08602976476236197,
      "grad_norm": 0.171875,
      "learning_rate": 8.582375478927202e-07,
      "loss": 0.015,
      "num_tokens": 39510307.0,
      "reward": 0.486979179084301,
      "reward_std": 0.15518230479210615,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.3542175292969,
      "epoch": 0.08641382621219396,
      "grad_norm": 0.2001953125,
      "learning_rate": 8.620689655172412e-07,
      "loss": 0.0887,
      "num_tokens": 39700757.0,
      "reward": 0.4192708358168602,
      "reward_std": 0.17781013995409012,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875074505806,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.9895935058594,
      "epoch": 0.08679788766202592,
      "grad_norm": 0.203125,
      "learning_rate": 8.659003831417624e-07,
      "loss": 0.1157,
      "num_tokens": 39886468.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.1479621483013034,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.2500610351562,
      "epoch": 0.08718194911185789,
      "grad_norm": 0.1943359375,
      "learning_rate": 8.697318007662835e-07,
      "loss": 0.0542,
      "num_tokens": 40063644.0,
      "reward": 0.4895833507180214,
      "reward_std": 0.21434927266091108,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1389.9896392822266,
      "epoch": 0.08756601056168987,
      "grad_norm": 0.1953125,
      "learning_rate": 8.735632183908046e-07,
      "loss": 0.0767,
      "num_tokens": 40210227.0,
      "reward": 0.486979179084301,
      "reward_std": 0.15865576080977917,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125074505806,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.8958740234375,
      "epoch": 0.08795007201152184,
      "grad_norm": 0.234375,
      "learning_rate": 8.773946360153256e-07,
      "loss": 0.0985,
      "num_tokens": 40384593.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.22777738235890865,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1444.9792175292969,
      "epoch": 0.08833413346135381,
      "grad_norm": 0.2451171875,
      "learning_rate": 8.812260536398467e-07,
      "loss": 0.0938,
      "num_tokens": 40537015.0,
      "reward": 0.6328125298023224,
      "reward_std": 0.3379022544249892,
      "rewards/accuracy_reward": 0.1979166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.4896240234375,
      "epoch": 0.0887181949111858,
      "grad_norm": 0.205078125,
      "learning_rate": 8.850574712643678e-07,
      "loss": 0.1168,
      "num_tokens": 40718054.0,
      "reward": 0.513020858168602,
      "reward_std": 0.2675612345337868,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541567325592,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.2709045410156,
      "epoch": 0.08910225636101776,
      "grad_norm": 0.1796875,
      "learning_rate": 8.888888888888888e-07,
      "loss": 0.0816,
      "num_tokens": 40894664.0,
      "reward": 0.4166666716337204,
      "reward_std": 0.13081647409126163,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.6875305175781,
      "epoch": 0.08948631781084973,
      "grad_norm": 0.16015625,
      "learning_rate": 8.927203065134099e-07,
      "loss": 0.0674,
      "num_tokens": 41076874.0,
      "reward": 0.40625,
      "reward_std": 0.1517950091511011,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666716337204,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.8542175292969,
      "epoch": 0.08987037926068171,
      "grad_norm": 0.2275390625,
      "learning_rate": 8.96551724137931e-07,
      "loss": 0.115,
      "num_tokens": 41243932.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.28932419046759605,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.9792175292969,
      "epoch": 0.09025444071051368,
      "grad_norm": 0.1806640625,
      "learning_rate": 9.003831417624521e-07,
      "loss": 0.1084,
      "num_tokens": 41412410.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.18197374045848846,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.1667175292969,
      "epoch": 0.09063850216034565,
      "grad_norm": 0.1943359375,
      "learning_rate": 9.042145593869731e-07,
      "loss": 0.0717,
      "num_tokens": 41602922.0,
      "reward": 0.3671875074505806,
      "reward_std": 0.10412419121712446,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541641831398,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.4375305175781,
      "epoch": 0.09102256361017763,
      "grad_norm": 0.1884765625,
      "learning_rate": 9.080459770114942e-07,
      "loss": 0.0899,
      "num_tokens": 41786332.0,
      "reward": 0.4322916716337204,
      "reward_std": 0.14672523643821478,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.9479675292969,
      "epoch": 0.0914066250600096,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.118773946360153e-07,
      "loss": 0.0984,
      "num_tokens": 41956119.0,
      "reward": 0.5572916865348816,
      "reward_std": 0.23004410322755575,
      "rewards/accuracy_reward": 0.14583333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.0416870117188,
      "epoch": 0.09179068650984157,
      "grad_norm": 0.2294921875,
      "learning_rate": 9.157088122605363e-07,
      "loss": 0.0608,
      "num_tokens": 42120675.0,
      "reward": 0.4244791865348816,
      "reward_std": 0.17877990659326315,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.6666870117188,
      "epoch": 0.09217474795967355,
      "grad_norm": 0.1904296875,
      "learning_rate": 9.195402298850574e-07,
      "loss": 0.0911,
      "num_tokens": 42307411.0,
      "reward": 0.3697916716337204,
      "reward_std": 0.13330329582095146,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1911.4583740234375,
      "epoch": 0.09255880940950552,
      "grad_norm": 0.154296875,
      "learning_rate": 9.233716475095785e-07,
      "loss": 0.063,
      "num_tokens": 42508727.0,
      "reward": 0.3489583507180214,
      "reward_std": 0.12257460784167051,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.328125,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.5208740234375,
      "epoch": 0.0929428708593375,
      "grad_norm": 0.1796875,
      "learning_rate": 9.272030651340996e-07,
      "loss": 0.0565,
      "num_tokens": 42685857.0,
      "reward": 0.5651041865348816,
      "reward_std": 0.32245301082730293,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1912.7396240234375,
      "epoch": 0.09332693230916947,
      "grad_norm": 0.1611328125,
      "learning_rate": 9.310344827586206e-07,
      "loss": 0.0456,
      "num_tokens": 42886072.0,
      "reward": 0.369791679084301,
      "reward_std": 0.10730221960693598,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3177083358168602,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.4167175292969,
      "epoch": 0.09371099375900144,
      "grad_norm": 0.16796875,
      "learning_rate": 9.348659003831417e-07,
      "loss": 0.0684,
      "num_tokens": 43061368.0,
      "reward": 0.4817708358168602,
      "reward_std": 0.1758501697331667,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875074505806,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.7083587646484,
      "epoch": 0.09409505520883342,
      "grad_norm": 0.267578125,
      "learning_rate": 9.386973180076628e-07,
      "loss": 0.0454,
      "num_tokens": 43215860.0,
      "reward": 0.5130208507180214,
      "reward_std": 0.27196265384554863,
      "rewards/accuracy_reward": 0.12500000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1893.7396240234375,
      "epoch": 0.09447911665866539,
      "grad_norm": 0.166015625,
      "learning_rate": 9.425287356321838e-07,
      "loss": 0.0695,
      "num_tokens": 43415963.0,
      "reward": 0.377604179084301,
      "reward_std": 0.09250425035133958,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3151041641831398,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.8229675292969,
      "epoch": 0.09486317810849736,
      "grad_norm": 0.1708984375,
      "learning_rate": 9.463601532567049e-07,
      "loss": 0.064,
      "num_tokens": 43598562.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.15485517214983702,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.1666870117188,
      "epoch": 0.09524723955832934,
      "grad_norm": 0.2021484375,
      "learning_rate": 9.50191570881226e-07,
      "loss": 0.0683,
      "num_tokens": 43774042.0,
      "reward": 0.5104166865348816,
      "reward_std": 0.15526545979082584,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.7500305175781,
      "epoch": 0.0956313010081613,
      "grad_norm": 0.236328125,
      "learning_rate": 9.540229885057471e-07,
      "loss": 0.1062,
      "num_tokens": 43943058.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.14301720540970564,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1738.0000610351562,
      "epoch": 0.09601536245799328,
      "grad_norm": 0.244140625,
      "learning_rate": 9.578544061302681e-07,
      "loss": 0.1425,
      "num_tokens": 44122378.0,
      "reward": 0.3567708358168602,
      "reward_std": 0.11561735905706882,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1705.0312805175781,
      "epoch": 0.09639942390782526,
      "grad_norm": 0.21484375,
      "learning_rate": 9.616858237547893e-07,
      "loss": 0.0572,
      "num_tokens": 44300717.0,
      "reward": 0.609375,
      "reward_std": 0.15540633350610733,
      "rewards/accuracy_reward": 0.2083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1662.6250610351562,
      "epoch": 0.09678348535765723,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.655172413793103e-07,
      "loss": 0.0607,
      "num_tokens": 44477249.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.13532005064189434,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.7187805175781,
      "epoch": 0.0971675468074892,
      "grad_norm": 0.2138671875,
      "learning_rate": 9.693486590038313e-07,
      "loss": 0.0788,
      "num_tokens": 44662366.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.20356722548604012,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.2812805175781,
      "epoch": 0.09755160825732118,
      "grad_norm": 0.1787109375,
      "learning_rate": 9.731800766283525e-07,
      "loss": 0.128,
      "num_tokens": 44852665.0,
      "reward": 0.369791679084301,
      "reward_std": 0.15755227208137512,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3593749925494194,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.3854370117188,
      "epoch": 0.09793566970715314,
      "grad_norm": 0.20703125,
      "learning_rate": 9.770114942528735e-07,
      "loss": 0.0957,
      "num_tokens": 45026270.0,
      "reward": 0.5937500149011612,
      "reward_std": 0.2205594964325428,
      "rewards/accuracy_reward": 0.1979166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.5312805175781,
      "epoch": 0.09831973115698511,
      "grad_norm": 0.1767578125,
      "learning_rate": 9.808429118773945e-07,
      "loss": 0.0717,
      "num_tokens": 45206081.0,
      "reward": 0.4713541865348816,
      "reward_std": 0.19455598294734955,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.53125,
      "epoch": 0.0987037926068171,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.846743295019157e-07,
      "loss": 0.1129,
      "num_tokens": 45375524.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.19804604910314083,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1535.9895935058594,
      "epoch": 0.09908785405664906,
      "grad_norm": 0.1953125,
      "learning_rate": 9.885057471264367e-07,
      "loss": 0.0407,
      "num_tokens": 45536451.0,
      "reward": 0.5494791939854622,
      "reward_std": 0.21955600008368492,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.2187805175781,
      "epoch": 0.09947191550648103,
      "grad_norm": 0.1845703125,
      "learning_rate": 9.92337164750958e-07,
      "loss": 0.0899,
      "num_tokens": 45721832.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.1879216618835926,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.3854370117188,
      "epoch": 0.09985597695631301,
      "grad_norm": 0.1767578125,
      "learning_rate": 9.961685823754787e-07,
      "loss": 0.1252,
      "num_tokens": 45899301.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.26617753878235817,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1702.4688110351562,
      "epoch": 0.10024003840614498,
      "grad_norm": 0.20703125,
      "learning_rate": 1e-06,
      "loss": 0.0555,
      "num_tokens": 46079154.0,
      "reward": 0.471354179084301,
      "reward_std": 0.16158428974449635,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.1041870117188,
      "epoch": 0.10062409985597695,
      "grad_norm": 0.1953125,
      "learning_rate": 9.999995951366943e-07,
      "loss": 0.091,
      "num_tokens": 46241580.0,
      "reward": 0.4661458358168602,
      "reward_std": 0.14294155267998576,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1873.1146240234375,
      "epoch": 0.10100816130580893,
      "grad_norm": 0.197265625,
      "learning_rate": 9.999983805475054e-07,
      "loss": 0.089,
      "num_tokens": 46433623.0,
      "reward": 0.377604179084301,
      "reward_std": 0.20770814269781113,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.6250305175781,
      "epoch": 0.1013922227556409,
      "grad_norm": 0.2138671875,
      "learning_rate": 9.999963562346187e-07,
      "loss": 0.1335,
      "num_tokens": 46613715.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.19557553716003895,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3750000074505806,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.1563110351562,
      "epoch": 0.10177628420547287,
      "grad_norm": 0.2353515625,
      "learning_rate": 9.999935222016772e-07,
      "loss": 0.1345,
      "num_tokens": 46792274.0,
      "reward": 0.471354179084301,
      "reward_std": 0.1733738947659731,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1726.0000610351562,
      "epoch": 0.10216034565530485,
      "grad_norm": 0.171875,
      "learning_rate": 9.999898784537802e-07,
      "loss": 0.1066,
      "num_tokens": 46971634.0,
      "reward": 0.377604179084301,
      "reward_std": 0.11600298620760441,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.5000610351562,
      "epoch": 0.10254440710513682,
      "grad_norm": 0.1748046875,
      "learning_rate": 9.999854249974842e-07,
      "loss": 0.0533,
      "num_tokens": 47164170.0,
      "reward": 0.3645833432674408,
      "reward_std": 0.1431873720139265,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3437499925494194,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.6979370117188,
      "epoch": 0.10292846855496879,
      "grad_norm": 0.1689453125,
      "learning_rate": 9.99980161840803e-07,
      "loss": 0.0485,
      "num_tokens": 47354685.0,
      "reward": 0.4453125223517418,
      "reward_std": 0.18694436457008123,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458283662796,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.7396240234375,
      "epoch": 0.10331253000480077,
      "grad_norm": 0.19921875,
      "learning_rate": 9.999740889932062e-07,
      "loss": 0.1372,
      "num_tokens": 47539244.0,
      "reward": 0.401041679084301,
      "reward_std": 0.18029431998729706,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.6771240234375,
      "epoch": 0.10369659145463274,
      "grad_norm": 0.2109375,
      "learning_rate": 9.999672064656225e-07,
      "loss": 0.1236,
      "num_tokens": 47724373.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.20853660628199577,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.2292175292969,
      "epoch": 0.10408065290446472,
      "grad_norm": 0.2392578125,
      "learning_rate": 9.999595142704352e-07,
      "loss": 0.0699,
      "num_tokens": 47898403.0,
      "reward": 0.4479166716337204,
      "reward_std": 0.22866962105035782,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.1145935058594,
      "epoch": 0.10446471435429669,
      "grad_norm": 0.17578125,
      "learning_rate": 9.99951012421486e-07,
      "loss": 0.0554,
      "num_tokens": 48074814.0,
      "reward": 0.4895833507180214,
      "reward_std": 0.18853413499891758,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.5729675292969,
      "epoch": 0.10484877580412866,
      "grad_norm": 0.193359375,
      "learning_rate": 9.999417009340734e-07,
      "loss": 0.032,
      "num_tokens": 48243717.0,
      "reward": 0.5468750149011612,
      "reward_std": 0.21707891300320625,
      "rewards/accuracy_reward": 0.13541666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.4271545410156,
      "epoch": 0.10523283725396064,
      "grad_norm": 0.19921875,
      "learning_rate": 9.99931579824952e-07,
      "loss": 0.1,
      "num_tokens": 48422222.0,
      "reward": 0.466145858168602,
      "reward_std": 0.19758474826812744,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.0416870117188,
      "epoch": 0.10561689870379261,
      "grad_norm": 0.208984375,
      "learning_rate": 9.999206491123335e-07,
      "loss": 0.1017,
      "num_tokens": 48594426.0,
      "reward": 0.5729166939854622,
      "reward_std": 0.2298596389591694,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.7812957763672,
      "epoch": 0.10600096015362458,
      "grad_norm": 0.189453125,
      "learning_rate": 9.999089088158868e-07,
      "loss": 0.0802,
      "num_tokens": 48751013.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.15109872445464134,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.7188110351562,
      "epoch": 0.10638502160345656,
      "grad_norm": 0.2021484375,
      "learning_rate": 9.998963589567376e-07,
      "loss": 0.0777,
      "num_tokens": 48931506.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.1737738880328834,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.7500305175781,
      "epoch": 0.10676908305328853,
      "grad_norm": 0.2109375,
      "learning_rate": 9.998829995574675e-07,
      "loss": 0.0716,
      "num_tokens": 49094434.0,
      "reward": 0.5781250149011612,
      "reward_std": 0.25345202907919884,
      "rewards/accuracy_reward": 0.15625000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.6042022705078,
      "epoch": 0.1071531445031205,
      "grad_norm": 0.423828125,
      "learning_rate": 9.998688306421155e-07,
      "loss": 0.0211,
      "num_tokens": 49251260.0,
      "reward": 0.494791679084301,
      "reward_std": 0.2196838054805994,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.2812805175781,
      "epoch": 0.10753720595295248,
      "grad_norm": 0.25390625,
      "learning_rate": 9.99853852236177e-07,
      "loss": 0.1055,
      "num_tokens": 49413719.0,
      "reward": 0.5546875149011612,
      "reward_std": 0.23952777683734894,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.5417175292969,
      "epoch": 0.10792126740278445,
      "grad_norm": 0.19140625,
      "learning_rate": 9.998380643666043e-07,
      "loss": 0.1338,
      "num_tokens": 49590299.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.20463109388947487,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.1667175292969,
      "epoch": 0.10830532885261641,
      "grad_norm": 0.1865234375,
      "learning_rate": 9.998214670618054e-07,
      "loss": 0.055,
      "num_tokens": 49778491.0,
      "reward": 0.4505208358168602,
      "reward_std": 0.2380858724936843,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1432.8542175292969,
      "epoch": 0.1086893903024484,
      "grad_norm": 0.2890625,
      "learning_rate": 9.99804060351646e-07,
      "loss": 0.1217,
      "num_tokens": 49930637.0,
      "reward": 0.5078125074505806,
      "reward_std": 0.15971057070419192,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.8542175292969,
      "epoch": 0.10907345175228036,
      "grad_norm": 0.1796875,
      "learning_rate": 9.997858442674473e-07,
      "loss": 0.0713,
      "num_tokens": 50121207.0,
      "reward": 0.3984375074505806,
      "reward_std": 0.2005210630595684,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375074505806,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.6562805175781,
      "epoch": 0.10945751320211233,
      "grad_norm": 0.255859375,
      "learning_rate": 9.99766818841987e-07,
      "loss": 0.1618,
      "num_tokens": 50292302.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.16308283433318138,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.0000305175781,
      "epoch": 0.10984157465194432,
      "grad_norm": 0.2177734375,
      "learning_rate": 9.997469841094997e-07,
      "loss": 0.092,
      "num_tokens": 50455398.0,
      "reward": 0.455729179084301,
      "reward_std": 0.1628522053360939,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.5312805175781,
      "epoch": 0.11022563610177628,
      "grad_norm": 0.2451171875,
      "learning_rate": 9.997263401056758e-07,
      "loss": -0.0186,
      "num_tokens": 50636321.0,
      "reward": 0.3932291865348816,
      "reward_std": 0.09442372340708971,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291567325592,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.6041870117188,
      "epoch": 0.11060969755160825,
      "grad_norm": 0.1923828125,
      "learning_rate": 9.997048868676615e-07,
      "loss": 0.1276,
      "num_tokens": 50814147.0,
      "reward": 0.4921875298023224,
      "reward_std": 0.2549152113497257,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.4687805175781,
      "epoch": 0.11099375900144023,
      "grad_norm": 0.169921875,
      "learning_rate": 9.996826244340603e-07,
      "loss": 0.0925,
      "num_tokens": 51000256.0,
      "reward": 0.3255208358168602,
      "reward_std": 0.08767688134685159,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3255208358168602,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.0208740234375,
      "epoch": 0.1113778204512722,
      "grad_norm": 0.2001953125,
      "learning_rate": 9.996595528449304e-07,
      "loss": 0.088,
      "num_tokens": 51171210.0,
      "reward": 0.486979179084301,
      "reward_std": 0.19388357922434807,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.9479370117188,
      "epoch": 0.11176188190110417,
      "grad_norm": 0.1943359375,
      "learning_rate": 9.996356721417871e-07,
      "loss": 0.0046,
      "num_tokens": 51355693.0,
      "reward": 0.3489583544433117,
      "reward_std": 0.14581995457410812,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3072916604578495,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.3646240234375,
      "epoch": 0.11214594335093615,
      "grad_norm": 0.193359375,
      "learning_rate": 9.996109823676008e-07,
      "loss": 0.0875,
      "num_tokens": 51545168.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.18138035014271736,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.947998046875,
      "epoch": 0.11253000480076812,
      "grad_norm": 0.18359375,
      "learning_rate": 9.995854835667985e-07,
      "loss": 0.086,
      "num_tokens": 51728747.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.2189176380634308,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166567325592,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.3646240234375,
      "epoch": 0.11291406625060009,
      "grad_norm": 0.169921875,
      "learning_rate": 9.995591757852622e-07,
      "loss": 0.0598,
      "num_tokens": 51914142.0,
      "reward": 0.479166679084301,
      "reward_std": 0.1152257607318461,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.34375,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.9583740234375,
      "epoch": 0.11329812770043207,
      "grad_norm": 0.1806640625,
      "learning_rate": 9.995320590703303e-07,
      "loss": 0.0966,
      "num_tokens": 52106562.0,
      "reward": 0.401041679084301,
      "reward_std": 0.15167571231722832,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583432674408,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.6667175292969,
      "epoch": 0.11368218915026404,
      "grad_norm": 0.1865234375,
      "learning_rate": 9.99504133470796e-07,
      "loss": 0.0559,
      "num_tokens": 52292914.0,
      "reward": 0.4322916716337204,
      "reward_std": 0.1828116998076439,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1422.9896240234375,
      "epoch": 0.11406625060009601,
      "grad_norm": 0.251953125,
      "learning_rate": 9.994753990369085e-07,
      "loss": 0.1523,
      "num_tokens": 52446145.0,
      "reward": 0.5286458432674408,
      "reward_std": 0.24964409694075584,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.4896240234375,
      "epoch": 0.11445031204992799,
      "grad_norm": 0.2216796875,
      "learning_rate": 9.994458558203725e-07,
      "loss": 0.0704,
      "num_tokens": 52617392.0,
      "reward": 0.502604179084301,
      "reward_std": 0.20371847227215767,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1870.1042175292969,
      "epoch": 0.11483437349975996,
      "grad_norm": 0.173828125,
      "learning_rate": 9.994155038743478e-07,
      "loss": 0.1123,
      "num_tokens": 52810578.0,
      "reward": 0.3411458358168602,
      "reward_std": 0.12646822445094585,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291716337204,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1491.1979675292969,
      "epoch": 0.11521843494959194,
      "grad_norm": 0.236328125,
      "learning_rate": 9.993843432534494e-07,
      "loss": 0.0727,
      "num_tokens": 52967701.0,
      "reward": 0.5338541865348816,
      "reward_std": 0.2290269653312862,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.7500610351562,
      "epoch": 0.11560249639942391,
      "grad_norm": 0.150390625,
      "learning_rate": 9.993523740137477e-07,
      "loss": 0.0432,
      "num_tokens": 53142093.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.1768146399408579,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708432674408,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.6666870117188,
      "epoch": 0.11598655784925588,
      "grad_norm": 0.1630859375,
      "learning_rate": 9.993195962127673e-07,
      "loss": 0.0697,
      "num_tokens": 53316109.0,
      "reward": 0.4713541865348816,
      "reward_std": 0.10930114146322012,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.9271545410156,
      "epoch": 0.11637061929908786,
      "grad_norm": 0.1669921875,
      "learning_rate": 9.99286009909489e-07,
      "loss": 0.0698,
      "num_tokens": 53495974.0,
      "reward": 0.4270833358168602,
      "reward_std": 0.15126446168869734,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.5833740234375,
      "epoch": 0.11675468074891983,
      "grad_norm": 0.2138671875,
      "learning_rate": 9.992516151643475e-07,
      "loss": 0.1231,
      "num_tokens": 53662566.0,
      "reward": 0.4869791716337204,
      "reward_std": 0.13887539133429527,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1797.8021545410156,
      "epoch": 0.1171387421987518,
      "grad_norm": 0.19140625,
      "learning_rate": 9.992164120392324e-07,
      "loss": 0.068,
      "num_tokens": 53850659.0,
      "reward": 0.4114583507180214,
      "reward_std": 0.14414987992495298,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.6771545410156,
      "epoch": 0.11752280364858378,
      "grad_norm": 0.26171875,
      "learning_rate": 9.991804005974879e-07,
      "loss": 0.096,
      "num_tokens": 54015628.0,
      "reward": 0.424479179084301,
      "reward_std": 0.1290290541946888,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.5104370117188,
      "epoch": 0.11790686509841575,
      "grad_norm": 0.2119140625,
      "learning_rate": 9.99143580903913e-07,
      "loss": 0.0673,
      "num_tokens": 54177005.0,
      "reward": 0.4817708358168602,
      "reward_std": 0.23045912012457848,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.5313110351562,
      "epoch": 0.11829092654824772,
      "grad_norm": 0.1962890625,
      "learning_rate": 9.991059530247602e-07,
      "loss": 0.0976,
      "num_tokens": 54360928.0,
      "reward": 0.377604179084301,
      "reward_std": 0.16650792956352234,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1678.4792175292969,
      "epoch": 0.1186749879980797,
      "grad_norm": 0.1884765625,
      "learning_rate": 9.990675170277375e-07,
      "loss": 0.086,
      "num_tokens": 54535654.0,
      "reward": 0.401041679084301,
      "reward_std": 0.1389285773038864,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.5312805175781,
      "epoch": 0.11905904944791167,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.990282729820057e-07,
      "loss": 0.0634,
      "num_tokens": 54702217.0,
      "reward": 0.5468750149011612,
      "reward_std": 0.2424054630100727,
      "rewards/accuracy_reward": 0.12500000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.6458740234375,
      "epoch": 0.11944311089774363,
      "grad_norm": 0.220703125,
      "learning_rate": 9.98988220958181e-07,
      "loss": 0.0906,
      "num_tokens": 54883903.0,
      "reward": 0.591145858168602,
      "reward_std": 0.23957921378314495,
      "rewards/accuracy_reward": 0.18750000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.6979675292969,
      "epoch": 0.11982717234757562,
      "grad_norm": 0.236328125,
      "learning_rate": 9.989473610283318e-07,
      "loss": 0.1214,
      "num_tokens": 55039746.0,
      "reward": 0.6041666716337204,
      "reward_std": 0.24187909439206123,
      "rewards/accuracy_reward": 0.16666666232049465,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4374999925494194,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.0312805175781,
      "epoch": 0.12021123379740759,
      "grad_norm": 0.1826171875,
      "learning_rate": 9.98905693265982e-07,
      "loss": 0.0807,
      "num_tokens": 55214557.0,
      "reward": 0.4140625149011612,
      "reward_std": 0.15781010128557682,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1884.5000305175781,
      "epoch": 0.12059529524723955,
      "grad_norm": 0.1611328125,
      "learning_rate": 9.988632177461077e-07,
      "loss": 0.0709,
      "num_tokens": 55411533.0,
      "reward": 0.338541679084301,
      "reward_std": 0.11784903472289443,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.328125,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.1666870117188,
      "epoch": 0.12097935669707154,
      "grad_norm": 0.2099609375,
      "learning_rate": 9.988199345451391e-07,
      "loss": 0.1288,
      "num_tokens": 55582549.0,
      "reward": 0.510416679084301,
      "reward_std": 0.2407328523695469,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.6770935058594,
      "epoch": 0.1213634181469035,
      "grad_norm": 0.1787109375,
      "learning_rate": 9.987758437409603e-07,
      "loss": 0.079,
      "num_tokens": 55765438.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.13116449676454067,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828124925494194,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.7812805175781,
      "epoch": 0.12174747959673547,
      "grad_norm": 0.240234375,
      "learning_rate": 9.98730945412907e-07,
      "loss": 0.0626,
      "num_tokens": 55942481.0,
      "reward": 0.3619791716337204,
      "reward_std": 0.12796014500781894,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.0208740234375,
      "epoch": 0.12213154104656745,
      "grad_norm": 0.20703125,
      "learning_rate": 9.986852396417693e-07,
      "loss": 0.0475,
      "num_tokens": 56111691.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.1976294293999672,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416567325592,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.7604370117188,
      "epoch": 0.12251560249639942,
      "grad_norm": 0.263671875,
      "learning_rate": 9.9863872650979e-07,
      "loss": 0.0944,
      "num_tokens": 56275916.0,
      "reward": 0.4765625223517418,
      "reward_std": 0.17187652736902237,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.5520935058594,
      "epoch": 0.12289966394623139,
      "grad_norm": 0.2236328125,
      "learning_rate": 9.985914061006646e-07,
      "loss": 0.1185,
      "num_tokens": 56458889.0,
      "reward": 0.3593750149011612,
      "reward_std": 0.10755525901913643,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3593749925494194,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.7083740234375,
      "epoch": 0.12328372539606337,
      "grad_norm": 0.1845703125,
      "learning_rate": 9.985432784995408e-07,
      "loss": 0.1147,
      "num_tokens": 56633229.0,
      "reward": 0.486979179084301,
      "reward_std": 0.19659107364714146,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1842.3854675292969,
      "epoch": 0.12366778684589534,
      "grad_norm": 0.171875,
      "learning_rate": 9.98494343793019e-07,
      "loss": 0.0831,
      "num_tokens": 56824362.0,
      "reward": 0.361979179084301,
      "reward_std": 0.15061912965029478,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1930.4062805175781,
      "epoch": 0.12405184829572731,
      "grad_norm": 0.1796875,
      "learning_rate": 9.984446020691518e-07,
      "loss": 0.0332,
      "num_tokens": 57026777.0,
      "reward": 0.3489583432674408,
      "reward_std": 0.11866077221930027,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416716337204,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.0208740234375,
      "epoch": 0.12443590974555929,
      "grad_norm": 0.13671875,
      "learning_rate": 9.983940534174445e-07,
      "loss": 0.0337,
      "num_tokens": 57218443.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.19705121126025915,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.6458740234375,
      "epoch": 0.12481997119539126,
      "grad_norm": 0.2451171875,
      "learning_rate": 9.983426979288538e-07,
      "loss": 0.1321,
      "num_tokens": 57399985.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.3098112568259239,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.2708740234375,
      "epoch": 0.12520403264522323,
      "grad_norm": 0.1806640625,
      "learning_rate": 9.982905356957886e-07,
      "loss": 0.058,
      "num_tokens": 57580059.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.1327183023095131,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.1666870117188,
      "epoch": 0.1255880940950552,
      "grad_norm": 0.22265625,
      "learning_rate": 9.982375668121086e-07,
      "loss": 0.1002,
      "num_tokens": 57743099.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.13717699516564608,
      "rewards/accuracy_reward": 0.08854166697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1733.7812805175781,
      "epoch": 0.1259721555448872,
      "grad_norm": 0.1845703125,
      "learning_rate": 9.98183791373126e-07,
      "loss": 0.0748,
      "num_tokens": 57923022.0,
      "reward": 0.447916679084301,
      "reward_std": 0.2051177527755499,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.4584045410156,
      "epoch": 0.12635621699471916,
      "grad_norm": 0.2197265625,
      "learning_rate": 9.981292094756041e-07,
      "loss": 0.0836,
      "num_tokens": 58090378.0,
      "reward": 0.5468750149011612,
      "reward_std": 0.24734482867643237,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.6146240234375,
      "epoch": 0.12674027844455113,
      "grad_norm": 0.169921875,
      "learning_rate": 9.98073821217757e-07,
      "loss": 0.0699,
      "num_tokens": 58276573.0,
      "reward": 0.354166679084301,
      "reward_std": 0.09127402631565928,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.6458435058594,
      "epoch": 0.1271243398943831,
      "grad_norm": 0.24609375,
      "learning_rate": 9.980176266992496e-07,
      "loss": 0.1339,
      "num_tokens": 58449531.0,
      "reward": 0.494791679084301,
      "reward_std": 0.15913371182978153,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.9375305175781,
      "epoch": 0.12750840134421507,
      "grad_norm": 0.240234375,
      "learning_rate": 9.979606260211986e-07,
      "loss": 0.1029,
      "num_tokens": 58634557.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.18058671802282333,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.5625305175781,
      "epoch": 0.12789246279404703,
      "grad_norm": 0.16796875,
      "learning_rate": 9.979028192861698e-07,
      "loss": 0.0643,
      "num_tokens": 58822355.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.16510222386568785,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1880.9896240234375,
      "epoch": 0.12827652424387903,
      "grad_norm": 0.1767578125,
      "learning_rate": 9.978442065981809e-07,
      "loss": 0.0803,
      "num_tokens": 59019058.0,
      "reward": 0.3333333507180214,
      "reward_std": 0.07886058557778597,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333283662796,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.3437805175781,
      "epoch": 0.128660585693711,
      "grad_norm": 0.279296875,
      "learning_rate": 9.977847880626985e-07,
      "loss": 0.0615,
      "num_tokens": 59188243.0,
      "reward": 0.4843750074505806,
      "reward_std": 0.2068762481212616,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.9063110351562,
      "epoch": 0.12904464714354297,
      "grad_norm": 0.181640625,
      "learning_rate": 9.977245637866404e-07,
      "loss": 0.0829,
      "num_tokens": 59358554.0,
      "reward": 0.5130208507180214,
      "reward_std": 0.19439666718244553,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.0000610351562,
      "epoch": 0.12942870859337494,
      "grad_norm": 0.197265625,
      "learning_rate": 9.976635338783733e-07,
      "loss": 0.0773,
      "num_tokens": 59545986.0,
      "reward": 0.3828125074505806,
      "reward_std": 0.1557865720242262,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1683.6666870117188,
      "epoch": 0.1298127700432069,
      "grad_norm": 0.203125,
      "learning_rate": 9.976016984477143e-07,
      "loss": 0.0928,
      "num_tokens": 59721754.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.21512503270059824,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875074505806,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.2292175292969,
      "epoch": 0.13019683149303887,
      "grad_norm": 0.1767578125,
      "learning_rate": 9.975390576059296e-07,
      "loss": 0.0519,
      "num_tokens": 59885368.0,
      "reward": 0.5078125074505806,
      "reward_std": 0.12153688818216324,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.9688110351562,
      "epoch": 0.13058089294287087,
      "grad_norm": 0.2177734375,
      "learning_rate": 9.97475611465734e-07,
      "loss": 0.1137,
      "num_tokens": 60047205.0,
      "reward": 0.5937500223517418,
      "reward_std": 0.2285020938143134,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.6458740234375,
      "epoch": 0.13096495439270284,
      "grad_norm": 0.185546875,
      "learning_rate": 9.97411360141293e-07,
      "loss": 0.0778,
      "num_tokens": 60225723.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.1642234455794096,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.3541870117188,
      "epoch": 0.1313490158425348,
      "grad_norm": 0.2099609375,
      "learning_rate": 9.973463037482193e-07,
      "loss": 0.1422,
      "num_tokens": 60410037.0,
      "reward": 0.4869791716337204,
      "reward_std": 0.22720644623041153,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.2917175292969,
      "epoch": 0.13173307729236677,
      "grad_norm": 0.2099609375,
      "learning_rate": 9.97280442403575e-07,
      "loss": 0.0707,
      "num_tokens": 60583985.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.21260273642838,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.3021240234375,
      "epoch": 0.13211713874219874,
      "grad_norm": 0.21484375,
      "learning_rate": 9.972137762258704e-07,
      "loss": 0.063,
      "num_tokens": 60759022.0,
      "reward": 0.5104166716337204,
      "reward_std": 0.2272140122950077,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1581.6146240234375,
      "epoch": 0.13250120019203074,
      "grad_norm": 0.1982421875,
      "learning_rate": 9.971463053350645e-07,
      "loss": 0.0706,
      "num_tokens": 60924233.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.1256679380312562,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1456.1667175292969,
      "epoch": 0.1328852616418627,
      "grad_norm": 0.26171875,
      "learning_rate": 9.970780298525636e-07,
      "loss": 0.079,
      "num_tokens": 61078409.0,
      "reward": 0.5052083358168602,
      "reward_std": 0.20232651010155678,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.3125305175781,
      "epoch": 0.13326932309169467,
      "grad_norm": 0.265625,
      "learning_rate": 9.970089499012222e-07,
      "loss": 0.1061,
      "num_tokens": 61242831.0,
      "reward": 0.4661458507180214,
      "reward_std": 0.21811760775744915,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1484.0312805175781,
      "epoch": 0.13365338454152664,
      "grad_norm": 0.2490234375,
      "learning_rate": 9.969390656053421e-07,
      "loss": 0.1467,
      "num_tokens": 61401066.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.14121088571846485,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1962.78125,
      "epoch": 0.1340374459913586,
      "grad_norm": 0.134765625,
      "learning_rate": 9.968683770906729e-07,
      "loss": 0.0689,
      "num_tokens": 61602549.0,
      "reward": 0.3411458432674408,
      "reward_std": 0.13941630721092224,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3098958358168602,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.6250305175781,
      "epoch": 0.13442150744119058,
      "grad_norm": 0.173828125,
      "learning_rate": 9.967968844844106e-07,
      "loss": 0.0672,
      "num_tokens": 61778881.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.14396959729492664,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1467.8854370117188,
      "epoch": 0.13480556889102258,
      "grad_norm": 0.1982421875,
      "learning_rate": 9.967245879151986e-07,
      "loss": 0.0613,
      "num_tokens": 61932950.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.21742641925811768,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041716337204,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.8646545410156,
      "epoch": 0.13518963034085454,
      "grad_norm": 0.1748046875,
      "learning_rate": 9.96651487513127e-07,
      "loss": 0.0756,
      "num_tokens": 62126249.0,
      "reward": 0.330729179084301,
      "reward_std": 0.11412729509174824,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3203125,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1442.8021240234375,
      "epoch": 0.1355736917906865,
      "grad_norm": 0.193359375,
      "learning_rate": 9.965775834097319e-07,
      "loss": 0.1034,
      "num_tokens": 62276766.0,
      "reward": 0.5859375223517418,
      "reward_std": 0.2626296356320381,
      "rewards/accuracy_reward": 0.16666666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1855.3229675292969,
      "epoch": 0.13595775324051848,
      "grad_norm": 0.1640625,
      "learning_rate": 9.96502875737996e-07,
      "loss": 0.0755,
      "num_tokens": 62470053.0,
      "reward": 0.3802083507180214,
      "reward_std": 0.12421102821826935,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.4271240234375,
      "epoch": 0.13634181469035045,
      "grad_norm": 0.1416015625,
      "learning_rate": 9.964273646323472e-07,
      "loss": 0.0726,
      "num_tokens": 62652758.0,
      "reward": 0.3489583432674408,
      "reward_std": 0.0905031212605536,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416641831398,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.71875,
      "epoch": 0.13672587614018242,
      "grad_norm": 0.19921875,
      "learning_rate": 9.963510502286601e-07,
      "loss": 0.1065,
      "num_tokens": 62832675.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.2392721027135849,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.9896240234375,
      "epoch": 0.1371099375900144,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.962739326642542e-07,
      "loss": 0.084,
      "num_tokens": 63004666.0,
      "reward": 0.526041679084301,
      "reward_std": 0.3003530204296112,
      "rewards/accuracy_reward": 0.12500000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.6041870117188,
      "epoch": 0.13749399903984638,
      "grad_norm": 0.2109375,
      "learning_rate": 9.961960120778942e-07,
      "loss": 0.1057,
      "num_tokens": 63189972.0,
      "reward": 0.3515625074505806,
      "reward_std": 0.11513411533087492,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.5625305175781,
      "epoch": 0.13787806048967835,
      "grad_norm": 0.2021484375,
      "learning_rate": 9.961172886097897e-07,
      "loss": 0.127,
      "num_tokens": 63379106.0,
      "reward": 0.3958333507180214,
      "reward_std": 0.16132488660514355,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333358168602,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.4688110351562,
      "epoch": 0.13826212193951032,
      "grad_norm": 0.1357421875,
      "learning_rate": 9.960377624015955e-07,
      "loss": 0.0671,
      "num_tokens": 63544847.0,
      "reward": 0.4244791865348816,
      "reward_std": 0.10652927681803703,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.6041870117188,
      "epoch": 0.1386461833893423,
      "grad_norm": 0.1796875,
      "learning_rate": 9.959574335964098e-07,
      "loss": 0.0983,
      "num_tokens": 63727321.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.17761795595288277,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.2188110351562,
      "epoch": 0.13903024483917426,
      "grad_norm": 0.2353515625,
      "learning_rate": 9.958763023387762e-07,
      "loss": 0.1204,
      "num_tokens": 63900342.0,
      "reward": 0.471354179084301,
      "reward_std": 0.17368455044925213,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1467.1770935058594,
      "epoch": 0.13941430628900625,
      "grad_norm": 0.2392578125,
      "learning_rate": 9.95794368774682e-07,
      "loss": 0.0426,
      "num_tokens": 64054447.0,
      "reward": 0.591145858168602,
      "reward_std": 0.26058339700102806,
      "rewards/accuracy_reward": 0.13541666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291641831398,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.8125305175781,
      "epoch": 0.13979836773883822,
      "grad_norm": 0.224609375,
      "learning_rate": 9.957116330515566e-07,
      "loss": 0.0982,
      "num_tokens": 64233253.0,
      "reward": 0.401041679084301,
      "reward_std": 0.13847165927290916,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.1041870117188,
      "epoch": 0.1401824291886702,
      "grad_norm": 0.20703125,
      "learning_rate": 9.956280953182752e-07,
      "loss": 0.0752,
      "num_tokens": 64394479.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.1166307870298624,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1439.3646240234375,
      "epoch": 0.14056649063850216,
      "grad_norm": 0.197265625,
      "learning_rate": 9.955437557251543e-07,
      "loss": 0.0604,
      "num_tokens": 64549186.0,
      "reward": 0.4531250074505806,
      "reward_std": 0.126278230920434,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.2187805175781,
      "epoch": 0.14095055208833412,
      "grad_norm": 0.166015625,
      "learning_rate": 9.954586144239544e-07,
      "loss": 0.0578,
      "num_tokens": 64733079.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.18550877273082733,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.9062805175781,
      "epoch": 0.1413346135381661,
      "grad_norm": 0.26171875,
      "learning_rate": 9.953726715678779e-07,
      "loss": 0.11,
      "num_tokens": 64896990.0,
      "reward": 0.5390625074505806,
      "reward_std": 0.22957831621170044,
      "rewards/accuracy_reward": 0.15624999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.5000610351562,
      "epoch": 0.1417186749879981,
      "grad_norm": 0.1806640625,
      "learning_rate": 9.952859273115695e-07,
      "loss": 0.1124,
      "num_tokens": 65073902.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.16044507548213005,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.4687805175781,
      "epoch": 0.14210273643783006,
      "grad_norm": 0.19921875,
      "learning_rate": 9.951983818111166e-07,
      "loss": 0.0728,
      "num_tokens": 65257723.0,
      "reward": 0.4322916865348816,
      "reward_std": 0.2156925704330206,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1440.7083435058594,
      "epoch": 0.14248679788766203,
      "grad_norm": 0.25390625,
      "learning_rate": 9.951100352240477e-07,
      "loss": 0.0701,
      "num_tokens": 65411895.0,
      "reward": 0.510416679084301,
      "reward_std": 0.22430717200040817,
      "rewards/accuracy_reward": 0.0677083320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.0729370117188,
      "epoch": 0.142870859337494,
      "grad_norm": 0.1689453125,
      "learning_rate": 9.95020887709333e-07,
      "loss": 0.0326,
      "num_tokens": 65603638.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.18034875392913818,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.0937805175781,
      "epoch": 0.14325492078732596,
      "grad_norm": 0.173828125,
      "learning_rate": 9.94930939427384e-07,
      "loss": 0.0916,
      "num_tokens": 65789695.0,
      "reward": 0.3671875074505806,
      "reward_std": 0.13250915240496397,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1678.1771240234375,
      "epoch": 0.14363898223715796,
      "grad_norm": 0.23828125,
      "learning_rate": 9.948401905400524e-07,
      "loss": 0.0629,
      "num_tokens": 65965248.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.11914495192468166,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1819.6667175292969,
      "epoch": 0.14402304368698993,
      "grad_norm": 0.189453125,
      "learning_rate": 9.947486412106318e-07,
      "loss": 0.0633,
      "num_tokens": 66154744.0,
      "reward": 0.4609375223517418,
      "reward_std": 0.23506863415241241,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.8854675292969,
      "epoch": 0.1444071051368219,
      "grad_norm": 0.1953125,
      "learning_rate": 9.946562916038548e-07,
      "loss": 0.032,
      "num_tokens": 66340485.0,
      "reward": 0.447916679084301,
      "reward_std": 0.21463812701404095,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3750000074505806,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.2396545410156,
      "epoch": 0.14479116658665386,
      "grad_norm": 0.23828125,
      "learning_rate": 9.945631418858952e-07,
      "loss": 0.0951,
      "num_tokens": 66508780.0,
      "reward": 0.5000000074505806,
      "reward_std": 0.2427233261987567,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.8750305175781,
      "epoch": 0.14517522803648583,
      "grad_norm": 0.228515625,
      "learning_rate": 9.944691922243653e-07,
      "loss": 0.1328,
      "num_tokens": 66672440.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.17896268144249916,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.9687805175781,
      "epoch": 0.1455592894863178,
      "grad_norm": 0.2099609375,
      "learning_rate": 9.943744427883177e-07,
      "loss": 0.1,
      "num_tokens": 66850573.0,
      "reward": 0.4166666716337204,
      "reward_std": 0.15960792358964682,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1875.4479370117188,
      "epoch": 0.1459433509361498,
      "grad_norm": 0.1708984375,
      "learning_rate": 9.942788937482436e-07,
      "loss": 0.0852,
      "num_tokens": 67044312.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.16503555653616786,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291641831398,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.3229370117188,
      "epoch": 0.14632741238598176,
      "grad_norm": 0.212890625,
      "learning_rate": 9.941825452760737e-07,
      "loss": -0.0049,
      "num_tokens": 67206119.0,
      "reward": 0.5885416716337204,
      "reward_std": 0.21013540774583817,
      "rewards/accuracy_reward": 0.18749999534338713,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.0000305175781,
      "epoch": 0.14671147383581373,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.940853975451765e-07,
      "loss": 0.082,
      "num_tokens": 67360951.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.25624422635883093,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.7813110351562,
      "epoch": 0.1470955352856457,
      "grad_norm": 0.28515625,
      "learning_rate": 9.939874507303583e-07,
      "loss": 0.1293,
      "num_tokens": 67523026.0,
      "reward": 0.4739583507180214,
      "reward_std": 0.217027198523283,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1263.5833740234375,
      "epoch": 0.14747959673547767,
      "grad_norm": 0.255859375,
      "learning_rate": 9.938887050078648e-07,
      "loss": 0.1364,
      "num_tokens": 67656770.0,
      "reward": 0.7343750149011612,
      "reward_std": 0.3058369569480419,
      "rewards/accuracy_reward": 0.17708333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5572916716337204,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.2917175292969,
      "epoch": 0.14786365818530964,
      "grad_norm": 0.216796875,
      "learning_rate": 9.937891605553779e-07,
      "loss": 0.1199,
      "num_tokens": 67821902.0,
      "reward": 0.432291679084301,
      "reward_std": 0.15540321543812752,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1704.3333435058594,
      "epoch": 0.14824771963514163,
      "grad_norm": 0.1943359375,
      "learning_rate": 9.936888175520168e-07,
      "loss": 0.0724,
      "num_tokens": 68000166.0,
      "reward": 0.3828125074505806,
      "reward_std": 0.10108048981055617,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.7188110351562,
      "epoch": 0.1486317810849736,
      "grad_norm": 0.220703125,
      "learning_rate": 9.935876761783384e-07,
      "loss": 0.1216,
      "num_tokens": 68172187.0,
      "reward": 0.6536458507180214,
      "reward_std": 0.3089594766497612,
      "rewards/accuracy_reward": 0.2500000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.0208435058594,
      "epoch": 0.14901584253480557,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.934857366163357e-07,
      "loss": 0.1451,
      "num_tokens": 68344805.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.2030621012672782,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.8333740234375,
      "epoch": 0.14939990398463754,
      "grad_norm": 0.203125,
      "learning_rate": 9.933829990494379e-07,
      "loss": 0.1179,
      "num_tokens": 68509773.0,
      "reward": 0.606770858168602,
      "reward_std": 0.26398175954818726,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875074505806,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.3229370117188,
      "epoch": 0.1497839654344695,
      "grad_norm": 0.1826171875,
      "learning_rate": 9.9327946366251e-07,
      "loss": 0.035,
      "num_tokens": 68675508.0,
      "reward": 0.494791679084301,
      "reward_std": 0.1966226426884532,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.6250305175781,
      "epoch": 0.15016802688430148,
      "grad_norm": 0.2197265625,
      "learning_rate": 9.93175130641853e-07,
      "loss": 0.0614,
      "num_tokens": 68840544.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.17684490606188774,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424479179084301,
      "step": 391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1634.1042175292969,
      "epoch": 0.15055208833413347,
      "grad_norm": 0.228515625,
      "learning_rate": 9.930700001752031e-07,
      "loss": 0.053,
      "num_tokens": 69010754.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.15090983267873526,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.4479370117188,
      "epoch": 0.15093614978396544,
      "grad_norm": 0.2041015625,
      "learning_rate": 9.929640724517311e-07,
      "loss": 0.1272,
      "num_tokens": 69185205.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.20136664807796478,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.9583740234375,
      "epoch": 0.1513202112337974,
      "grad_norm": 0.21484375,
      "learning_rate": 9.928573476620425e-07,
      "loss": 0.1009,
      "num_tokens": 69359393.0,
      "reward": 0.4479166716337204,
      "reward_std": 0.22123561054468155,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1865.2187805175781,
      "epoch": 0.15170427268362938,
      "grad_norm": 0.1708984375,
      "learning_rate": 9.92749825998177e-07,
      "loss": 0.0697,
      "num_tokens": 69553054.0,
      "reward": 0.3515625074505806,
      "reward_std": 0.12709845881909132,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.2396240234375,
      "epoch": 0.15208833413346134,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.926415076536085e-07,
      "loss": 0.0736,
      "num_tokens": 69738989.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.16790299024432898,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.4791870117188,
      "epoch": 0.1524723955832933,
      "grad_norm": 0.220703125,
      "learning_rate": 9.92532392823244e-07,
      "loss": 0.0726,
      "num_tokens": 69920011.0,
      "reward": 0.4036458507180214,
      "reward_std": 0.13934841519221663,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.0937805175781,
      "epoch": 0.1528564570331253,
      "grad_norm": 0.1865234375,
      "learning_rate": 9.92422481703424e-07,
      "loss": 0.1049,
      "num_tokens": 70092316.0,
      "reward": 0.4791666716337204,
      "reward_std": 0.25783059746026993,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.0937805175781,
      "epoch": 0.15324051848295728,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.92311774491922e-07,
      "loss": 0.1337,
      "num_tokens": 70253109.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.1636130828410387,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416567325592,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.2708740234375,
      "epoch": 0.15362457993278925,
      "grad_norm": 0.1962890625,
      "learning_rate": 9.922002713879435e-07,
      "loss": 0.1201,
      "num_tokens": 70423439.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.11173602379858494,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1509.8438110351562,
      "epoch": 0.15400864138262121,
      "grad_norm": 0.21875,
      "learning_rate": 9.92087972592126e-07,
      "loss": 0.091,
      "num_tokens": 70580784.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.2347719632089138,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1484.1979370117188,
      "epoch": 0.15439270283245318,
      "grad_norm": 0.1728515625,
      "learning_rate": 9.919748783065394e-07,
      "loss": 0.1167,
      "num_tokens": 70736187.0,
      "reward": 0.494791679084301,
      "reward_std": 0.20071565732359886,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.1458435058594,
      "epoch": 0.15477676428228518,
      "grad_norm": 0.203125,
      "learning_rate": 9.91860988734685e-07,
      "loss": 0.0538,
      "num_tokens": 70907625.0,
      "reward": 0.5234375074505806,
      "reward_std": 0.2575695812702179,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.9062805175781,
      "epoch": 0.15516082573211715,
      "grad_norm": 0.177734375,
      "learning_rate": 9.917463040814945e-07,
      "loss": 0.0405,
      "num_tokens": 71091432.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.13126103207468987,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416641831398,
      "step": 404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.9063110351562,
      "epoch": 0.15554488718194912,
      "grad_norm": 0.19140625,
      "learning_rate": 9.916308245533306e-07,
      "loss": 0.0795,
      "num_tokens": 71274703.0,
      "reward": 0.4270833358168602,
      "reward_std": 0.23436661437153816,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.9687805175781,
      "epoch": 0.15592894863178108,
      "grad_norm": 0.2060546875,
      "learning_rate": 9.915145503579864e-07,
      "loss": 0.0546,
      "num_tokens": 71454612.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.2405456369742751,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.5417175292969,
      "epoch": 0.15631301008161305,
      "grad_norm": 0.19140625,
      "learning_rate": 9.91397481704685e-07,
      "loss": 0.0915,
      "num_tokens": 71619536.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.14356949552893639,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.4583740234375,
      "epoch": 0.15669707153144502,
      "grad_norm": 0.1767578125,
      "learning_rate": 9.912796188040783e-07,
      "loss": 0.0878,
      "num_tokens": 71800284.0,
      "reward": 0.3333333432674408,
      "reward_std": 0.05024262797087431,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333358168602,
      "step": 408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.7604370117188,
      "epoch": 0.15708113298127702,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.911609618682487e-07,
      "loss": 0.04,
      "num_tokens": 71981245.0,
      "reward": 0.4114583358168602,
      "reward_std": 0.14900817349553108,
      "rewards/accuracy_reward": 0.057291666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.4583435058594,
      "epoch": 0.15746519443110898,
      "grad_norm": 0.2138671875,
      "learning_rate": 9.910415111107058e-07,
      "loss": 0.0898,
      "num_tokens": 72162857.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.1664278320968151,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.2916870117188,
      "epoch": 0.15784925588094095,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.909212667463887e-07,
      "loss": 0.1137,
      "num_tokens": 72338789.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.19035021029412746,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1886.6146240234375,
      "epoch": 0.15823331733077292,
      "grad_norm": 0.1962890625,
      "learning_rate": 9.908002289916645e-07,
      "loss": 0.0832,
      "num_tokens": 72535176.0,
      "reward": 0.4401041716337204,
      "reward_std": 0.2153205405920744,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375,
      "step": 412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.4062805175781,
      "epoch": 0.1586173787806049,
      "grad_norm": 0.23046875,
      "learning_rate": 9.906783980643272e-07,
      "loss": 0.1207,
      "num_tokens": 72701207.0,
      "reward": 0.5729166716337204,
      "reward_std": 0.29139479994773865,
      "rewards/accuracy_reward": 0.16666667070239782,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.2500305175781,
      "epoch": 0.15900144023043686,
      "grad_norm": 0.23828125,
      "learning_rate": 9.905557741835988e-07,
      "loss": 0.1058,
      "num_tokens": 72867959.0,
      "reward": 0.6145833432674408,
      "reward_std": 0.23736819252371788,
      "rewards/accuracy_reward": 0.22916666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1679.9687805175781,
      "epoch": 0.15938550168026885,
      "grad_norm": 0.2001953125,
      "learning_rate": 9.904323575701273e-07,
      "loss": 0.0932,
      "num_tokens": 73043668.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.26498596277087927,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.6979370117188,
      "epoch": 0.15976956313010082,
      "grad_norm": 0.1787109375,
      "learning_rate": 9.90308148445988e-07,
      "loss": 0.086,
      "num_tokens": 73209063.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.27362002432346344,
      "rewards/accuracy_reward": 0.10416666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.6146240234375,
      "epoch": 0.1601536245799328,
      "grad_norm": 0.2060546875,
      "learning_rate": 9.90183147034682e-07,
      "loss": 0.1069,
      "num_tokens": 73378610.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.34739041328430176,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.1458740234375,
      "epoch": 0.16053768602976476,
      "grad_norm": 0.1845703125,
      "learning_rate": 9.900573535611357e-07,
      "loss": 0.1184,
      "num_tokens": 73559144.0,
      "reward": 0.4609375223517418,
      "reward_std": 0.19934247620403767,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1338.1354522705078,
      "epoch": 0.16092174747959673,
      "grad_norm": 0.275390625,
      "learning_rate": 9.899307682517008e-07,
      "loss": 0.0614,
      "num_tokens": 73700085.0,
      "reward": 0.5312500223517418,
      "reward_std": 0.16266787704080343,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.0000305175781,
      "epoch": 0.1613058089294287,
      "grad_norm": 0.2060546875,
      "learning_rate": 9.89803391334154e-07,
      "loss": 0.1286,
      "num_tokens": 73883973.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.20347534772008657,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1417.9062957763672,
      "epoch": 0.1616898703792607,
      "grad_norm": 0.2412109375,
      "learning_rate": 9.896752230376966e-07,
      "loss": 0.1399,
      "num_tokens": 74032252.0,
      "reward": 0.6041666865348816,
      "reward_std": 0.2858721688389778,
      "rewards/accuracy_reward": 0.14583333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333283662796,
      "step": 421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.0104370117188,
      "epoch": 0.16207393182909266,
      "grad_norm": 0.2373046875,
      "learning_rate": 9.895462635929534e-07,
      "loss": 0.1669,
      "num_tokens": 74197101.0,
      "reward": 0.5260416716337204,
      "reward_std": 0.2621201975271106,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.6458740234375,
      "epoch": 0.16245799327892463,
      "grad_norm": 0.212890625,
      "learning_rate": 9.894165132319732e-07,
      "loss": 0.0677,
      "num_tokens": 74377203.0,
      "reward": 0.5338541716337204,
      "reward_std": 0.27608904242515564,
      "rewards/accuracy_reward": 0.15624999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.4791870117188,
      "epoch": 0.1628420547287566,
      "grad_norm": 0.1787109375,
      "learning_rate": 9.89285972188228e-07,
      "loss": 0.1018,
      "num_tokens": 74563001.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.20299864001572132,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1850.5833740234375,
      "epoch": 0.16322611617858857,
      "grad_norm": 0.15234375,
      "learning_rate": 9.89154640696612e-07,
      "loss": 0.0605,
      "num_tokens": 74753857.0,
      "reward": 0.390625,
      "reward_std": 0.19240239635109901,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.4167175292969,
      "epoch": 0.16361017762842056,
      "grad_norm": 0.208984375,
      "learning_rate": 9.890225189934423e-07,
      "loss": 0.0972,
      "num_tokens": 74924241.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.20722655951976776,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.8541870117188,
      "epoch": 0.16399423907825253,
      "grad_norm": 0.2099609375,
      "learning_rate": 9.888896073164575e-07,
      "loss": 0.07,
      "num_tokens": 75087219.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.21931840851902962,
      "rewards/accuracy_reward": 0.11458333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.8438110351562,
      "epoch": 0.1643783005280845,
      "grad_norm": 0.1796875,
      "learning_rate": 9.887559059048182e-07,
      "loss": 0.0176,
      "num_tokens": 75276020.0,
      "reward": 0.432291679084301,
      "reward_std": 0.13593238545581698,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583283662796,
      "step": 428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1677.7813110351562,
      "epoch": 0.16476236197791647,
      "grad_norm": 0.1884765625,
      "learning_rate": 9.886214149991056e-07,
      "loss": 0.0863,
      "num_tokens": 75448567.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.22796825133264065,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.5000305175781,
      "epoch": 0.16514642342774843,
      "grad_norm": 0.17578125,
      "learning_rate": 9.884861348413214e-07,
      "loss": 0.1074,
      "num_tokens": 75629575.0,
      "reward": 0.447916679084301,
      "reward_std": 0.1456922758370638,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.3750610351562,
      "epoch": 0.1655304848775804,
      "grad_norm": 0.22265625,
      "learning_rate": 9.883500656748879e-07,
      "loss": 0.0821,
      "num_tokens": 75803379.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.20946595631539822,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.9166870117188,
      "epoch": 0.1659145463274124,
      "grad_norm": 0.2451171875,
      "learning_rate": 9.882132077446469e-07,
      "loss": 0.076,
      "num_tokens": 75975923.0,
      "reward": 0.5364583507180214,
      "reward_std": 0.22500597685575485,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.9479370117188,
      "epoch": 0.16629860777724437,
      "grad_norm": 0.19140625,
      "learning_rate": 9.880755612968593e-07,
      "loss": 0.0706,
      "num_tokens": 76151478.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.1796796079725027,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.7709045410156,
      "epoch": 0.16668266922707634,
      "grad_norm": 0.201171875,
      "learning_rate": 9.879371265792057e-07,
      "loss": 0.089,
      "num_tokens": 76314272.0,
      "reward": 0.4895833507180214,
      "reward_std": 0.2033696984872222,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1558.9375610351562,
      "epoch": 0.1670667306769083,
      "grad_norm": 0.216796875,
      "learning_rate": 9.87797903840784e-07,
      "loss": 0.0525,
      "num_tokens": 76475986.0,
      "reward": 0.5963541716337204,
      "reward_std": 0.23095927387475967,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1740.4375610351562,
      "epoch": 0.16745079212674027,
      "grad_norm": 0.2021484375,
      "learning_rate": 9.876578933321103e-07,
      "loss": 0.0896,
      "num_tokens": 76657436.0,
      "reward": 0.536458358168602,
      "reward_std": 0.3362790681421757,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.1041870117188,
      "epoch": 0.16783485357657224,
      "grad_norm": 0.2216796875,
      "learning_rate": 9.875170953051192e-07,
      "loss": 0.0443,
      "num_tokens": 76819086.0,
      "reward": 0.471354179084301,
      "reward_std": 0.18233572132885456,
      "rewards/accuracy_reward": 0.046875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.9271545410156,
      "epoch": 0.16821891502640424,
      "grad_norm": 0.2421875,
      "learning_rate": 9.873755100131612e-07,
      "loss": 0.0728,
      "num_tokens": 76988015.0,
      "reward": 0.4479166716337204,
      "reward_std": 0.22402901202440262,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1673.8645935058594,
      "epoch": 0.1686029764762362,
      "grad_norm": 0.19921875,
      "learning_rate": 9.872331377110036e-07,
      "loss": 0.0274,
      "num_tokens": 77163194.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.14080113545060158,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906250074505806,
      "step": 439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1474.0312805175781,
      "epoch": 0.16898703792606817,
      "grad_norm": 0.224609375,
      "learning_rate": 9.870899786548303e-07,
      "loss": 0.0042,
      "num_tokens": 77317237.0,
      "reward": 0.6979167014360428,
      "reward_std": 0.25254886969923973,
      "rewards/accuracy_reward": 0.22916667349636555,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.9687805175781,
      "epoch": 0.16937109937590014,
      "grad_norm": 0.1923828125,
      "learning_rate": 9.869460331022405e-07,
      "loss": 0.0631,
      "num_tokens": 77486538.0,
      "reward": 0.5677083358168602,
      "reward_std": 0.24004046991467476,
      "rewards/accuracy_reward": 0.14583333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.2813110351562,
      "epoch": 0.1697551608257321,
      "grad_norm": 0.1748046875,
      "learning_rate": 9.86801301312249e-07,
      "loss": 0.0745,
      "num_tokens": 77669557.0,
      "reward": 0.408854179084301,
      "reward_std": 0.1399763599038124,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1691.1354370117188,
      "epoch": 0.17013922227556408,
      "grad_norm": 0.228515625,
      "learning_rate": 9.86655783545285e-07,
      "loss": 0.0563,
      "num_tokens": 77847050.0,
      "reward": 0.440104179084301,
      "reward_std": 0.15229284949600697,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671874925494194,
      "step": 443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.7813110351562,
      "epoch": 0.17052328372539607,
      "grad_norm": 0.177734375,
      "learning_rate": 9.86509480063192e-07,
      "loss": 0.0984,
      "num_tokens": 78028877.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.1903197867795825,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.2917175292969,
      "epoch": 0.17090734517522804,
      "grad_norm": 0.19921875,
      "learning_rate": 9.863623911292272e-07,
      "loss": 0.0466,
      "num_tokens": 78204225.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.21903618425130844,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.2187805175781,
      "epoch": 0.17129140662506,
      "grad_norm": 0.2021484375,
      "learning_rate": 9.862145170080617e-07,
      "loss": 0.1094,
      "num_tokens": 78389814.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.21046173386275768,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.5625305175781,
      "epoch": 0.17167546807489198,
      "grad_norm": 0.2109375,
      "learning_rate": 9.860658579657786e-07,
      "loss": 0.1305,
      "num_tokens": 78577092.0,
      "reward": 0.4348958358168602,
      "reward_std": 0.18309490755200386,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.9375610351562,
      "epoch": 0.17205952952472395,
      "grad_norm": 0.22265625,
      "learning_rate": 9.859164142698742e-07,
      "loss": 0.0615,
      "num_tokens": 78743726.0,
      "reward": 0.3984375223517418,
      "reward_std": 0.15406453795731068,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1886.0313110351562,
      "epoch": 0.17244359097455592,
      "grad_norm": 0.19921875,
      "learning_rate": 9.857661861892565e-07,
      "loss": 0.0574,
      "num_tokens": 78941793.0,
      "reward": 0.3984375074505806,
      "reward_std": 0.19328990206122398,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375,
      "step": 449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.0938110351562,
      "epoch": 0.1728276524243879,
      "grad_norm": 0.2265625,
      "learning_rate": 9.856151739942445e-07,
      "loss": 0.0922,
      "num_tokens": 79110018.0,
      "reward": 0.5807291865348816,
      "reward_std": 0.1575652901083231,
      "rewards/accuracy_reward": 0.1666666716337204,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1642.0417175292969,
      "epoch": 0.17321171387421988,
      "grad_norm": 0.212890625,
      "learning_rate": 9.854633779565684e-07,
      "loss": 0.173,
      "num_tokens": 79282630.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.2812185473740101,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.6667175292969,
      "epoch": 0.17359577532405185,
      "grad_norm": 0.1689453125,
      "learning_rate": 9.853107983493692e-07,
      "loss": 0.0463,
      "num_tokens": 79470518.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.162523552775383,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416716337204,
      "step": 452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.3750610351562,
      "epoch": 0.17397983677388382,
      "grad_norm": 0.15625,
      "learning_rate": 9.85157435447197e-07,
      "loss": 0.0965,
      "num_tokens": 79658922.0,
      "reward": 0.3567708432674408,
      "reward_std": 0.12336704879999161,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1896.8125305175781,
      "epoch": 0.17436389822371579,
      "grad_norm": 0.16015625,
      "learning_rate": 9.850032895260122e-07,
      "loss": 0.077,
      "num_tokens": 79858480.0,
      "reward": 0.3333333432674408,
      "reward_std": 0.12053176388144493,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3229166641831398,
      "step": 454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1487.1771087646484,
      "epoch": 0.17474795967354778,
      "grad_norm": 0.234375,
      "learning_rate": 9.848483608631837e-07,
      "loss": 0.0974,
      "num_tokens": 80018489.0,
      "reward": 0.510416679084301,
      "reward_std": 0.213111724704504,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333283662796,
      "step": 455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.4271240234375,
      "epoch": 0.17513202112337975,
      "grad_norm": 0.2255859375,
      "learning_rate": 9.84692649737489e-07,
      "loss": 0.1091,
      "num_tokens": 80202618.0,
      "reward": 0.3750000074505806,
      "reward_std": 0.10657897032797337,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.6354675292969,
      "epoch": 0.17551608257321172,
      "grad_norm": 0.251953125,
      "learning_rate": 9.845361564291132e-07,
      "loss": 0.0941,
      "num_tokens": 80367551.0,
      "reward": 0.6015625149011612,
      "reward_std": 0.36475885286927223,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.8229522705078,
      "epoch": 0.1759001440230437,
      "grad_norm": 0.263671875,
      "learning_rate": 9.843788812196498e-07,
      "loss": 0.1263,
      "num_tokens": 80524046.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.1889595054090023,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.8958740234375,
      "epoch": 0.17628420547287565,
      "grad_norm": 0.1865234375,
      "learning_rate": 9.84220824392098e-07,
      "loss": 0.0877,
      "num_tokens": 80687500.0,
      "reward": 0.5104166865348816,
      "reward_std": 0.2033597119152546,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1463.0729675292969,
      "epoch": 0.17666826692270762,
      "grad_norm": 0.1943359375,
      "learning_rate": 9.840619862308647e-07,
      "loss": 0.067,
      "num_tokens": 80840955.0,
      "reward": 0.4244791865348816,
      "reward_std": 0.1230400362983346,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1705.9271545410156,
      "epoch": 0.17705232837253962,
      "grad_norm": 0.2431640625,
      "learning_rate": 9.839023670217616e-07,
      "loss": 0.1423,
      "num_tokens": 81017860.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.152727410197258,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.8958740234375,
      "epoch": 0.1774363898223716,
      "grad_norm": 0.228515625,
      "learning_rate": 9.837419670520064e-07,
      "loss": 0.1157,
      "num_tokens": 81205994.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.1746551301330328,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1677.6146240234375,
      "epoch": 0.17782045127220356,
      "grad_norm": 0.259765625,
      "learning_rate": 9.835807866102218e-07,
      "loss": 0.1534,
      "num_tokens": 81381877.0,
      "reward": 0.3645833432674408,
      "reward_std": 0.13287289533764124,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666716337204,
      "step": 463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.6250305175781,
      "epoch": 0.17820451272203552,
      "grad_norm": 0.203125,
      "learning_rate": 9.834188259864344e-07,
      "loss": 0.0428,
      "num_tokens": 81549849.0,
      "reward": 0.440104179084301,
      "reward_std": 0.14926113188266754,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.1771240234375,
      "epoch": 0.1785885741718675,
      "grad_norm": 0.2119140625,
      "learning_rate": 9.832560854720752e-07,
      "loss": 0.082,
      "num_tokens": 81703754.0,
      "reward": 0.5937500298023224,
      "reward_std": 0.23749571200460196,
      "rewards/accuracy_reward": 0.17708333022892475,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.1979675292969,
      "epoch": 0.17897263562169946,
      "grad_norm": 0.197265625,
      "learning_rate": 9.830925653599782e-07,
      "loss": 0.1293,
      "num_tokens": 81884789.0,
      "reward": 0.479166679084301,
      "reward_std": 0.20150185655802488,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.4791870117188,
      "epoch": 0.17935669707153146,
      "grad_norm": 0.2333984375,
      "learning_rate": 9.8292826594438e-07,
      "loss": 0.122,
      "num_tokens": 82057179.0,
      "reward": 0.549479179084301,
      "reward_std": 0.28308643214404583,
      "rewards/accuracy_reward": 0.1562499962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.2396240234375,
      "epoch": 0.17974075852136343,
      "grad_norm": 0.185546875,
      "learning_rate": 9.827631875209199e-07,
      "loss": 0.1092,
      "num_tokens": 82246434.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.2829824909567833,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458283662796,
      "step": 468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.1667175292969,
      "epoch": 0.1801248199711954,
      "grad_norm": 0.232421875,
      "learning_rate": 9.825973303866388e-07,
      "loss": 0.0811,
      "num_tokens": 82415890.0,
      "reward": 0.4583333507180214,
      "reward_std": 0.15103021916002035,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.7500305175781,
      "epoch": 0.18050888142102736,
      "grad_norm": 0.2177734375,
      "learning_rate": 9.824306948399787e-07,
      "loss": 0.1154,
      "num_tokens": 82600098.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.21126993000507355,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.4271545410156,
      "epoch": 0.18089294287085933,
      "grad_norm": 0.173828125,
      "learning_rate": 9.822632811807825e-07,
      "loss": 0.1134,
      "num_tokens": 82783787.0,
      "reward": 0.3697916716337204,
      "reward_std": 0.11617766693234444,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.6354675292969,
      "epoch": 0.1812770043206913,
      "grad_norm": 0.19921875,
      "learning_rate": 9.820950897102928e-07,
      "loss": 0.0517,
      "num_tokens": 82966480.0,
      "reward": 0.385416679084301,
      "reward_std": 0.09622385120019317,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1844.4063110351562,
      "epoch": 0.1816610657705233,
      "grad_norm": 0.2216796875,
      "learning_rate": 9.819261207311524e-07,
      "loss": 0.046,
      "num_tokens": 83156591.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.22495095059275627,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.0521240234375,
      "epoch": 0.18204512722035526,
      "grad_norm": 0.185546875,
      "learning_rate": 9.817563745474027e-07,
      "loss": 0.0958,
      "num_tokens": 83340836.0,
      "reward": 0.377604179084301,
      "reward_std": 0.12652720976620913,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375,
      "step": 474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.1562805175781,
      "epoch": 0.18242918867018723,
      "grad_norm": 0.2197265625,
      "learning_rate": 9.815858514644835e-07,
      "loss": 0.072,
      "num_tokens": 83517019.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.20145583152770996,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.3645935058594,
      "epoch": 0.1828132501200192,
      "grad_norm": 0.267578125,
      "learning_rate": 9.81414551789233e-07,
      "loss": -0.0071,
      "num_tokens": 83709198.0,
      "reward": 0.3671875074505806,
      "reward_std": 0.10510488227009773,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375,
      "step": 476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.9479370117188,
      "epoch": 0.18319731156985117,
      "grad_norm": 0.205078125,
      "learning_rate": 9.812424758298865e-07,
      "loss": 0.0737,
      "num_tokens": 83880865.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.1855965917930007,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.0,
      "epoch": 0.18358137301968314,
      "grad_norm": 0.2265625,
      "learning_rate": 9.810696238960762e-07,
      "loss": 0.0511,
      "num_tokens": 84066937.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.22566423518583179,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.8646240234375,
      "epoch": 0.18396543446951513,
      "grad_norm": 0.1806640625,
      "learning_rate": 9.808959962988303e-07,
      "loss": 0.0298,
      "num_tokens": 84249068.0,
      "reward": 0.3750000074505806,
      "reward_std": 0.1286996230483055,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666716337204,
      "step": 479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.8750305175781,
      "epoch": 0.1843494959193471,
      "grad_norm": 0.1865234375,
      "learning_rate": 9.807215933505735e-07,
      "loss": 0.1084,
      "num_tokens": 84439096.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.2144331894814968,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.1458740234375,
      "epoch": 0.18473355736917907,
      "grad_norm": 0.2373046875,
      "learning_rate": 9.805464153651247e-07,
      "loss": 0.0647,
      "num_tokens": 84609398.0,
      "reward": 0.4895833432674408,
      "reward_std": 0.22111480496823788,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.0104370117188,
      "epoch": 0.18511761881901104,
      "grad_norm": 0.236328125,
      "learning_rate": 9.803704626576978e-07,
      "loss": 0.0619,
      "num_tokens": 84778087.0,
      "reward": 0.5416666865348816,
      "reward_std": 0.22988194599747658,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.5104675292969,
      "epoch": 0.185501680268843,
      "grad_norm": 0.1953125,
      "learning_rate": 9.801937355449012e-07,
      "loss": 0.1159,
      "num_tokens": 84950528.0,
      "reward": 0.3854166716337204,
      "reward_std": 0.10701190680265427,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.9375610351562,
      "epoch": 0.185885741718675,
      "grad_norm": 0.201171875,
      "learning_rate": 9.800162343447362e-07,
      "loss": 0.112,
      "num_tokens": 85119818.0,
      "reward": 0.486979179084301,
      "reward_std": 0.19306168146431446,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1852.3958435058594,
      "epoch": 0.18626980316850697,
      "grad_norm": 0.2109375,
      "learning_rate": 9.798379593765969e-07,
      "loss": 0.0945,
      "num_tokens": 85312472.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.22899225540459156,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.0625610351562,
      "epoch": 0.18665386461833894,
      "grad_norm": 0.169921875,
      "learning_rate": 9.796589109612699e-07,
      "loss": 0.0668,
      "num_tokens": 85494022.0,
      "reward": 0.5364583432674408,
      "reward_std": 0.217063098680228,
      "rewards/accuracy_reward": 0.17708333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3593749925494194,
      "step": 486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1456.9375610351562,
      "epoch": 0.1870379260681709,
      "grad_norm": 0.189453125,
      "learning_rate": 9.79479089420934e-07,
      "loss": 0.0687,
      "num_tokens": 85647032.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.14715704135596752,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333432674408,
      "step": 487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.6667022705078,
      "epoch": 0.18742198751800287,
      "grad_norm": 0.1767578125,
      "learning_rate": 9.792984950791584e-07,
      "loss": 0.1046,
      "num_tokens": 85813112.0,
      "reward": 0.510416679084301,
      "reward_std": 0.12993250973522663,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.2708740234375,
      "epoch": 0.18780604896783484,
      "grad_norm": 0.21484375,
      "learning_rate": 9.791171282609033e-07,
      "loss": 0.0867,
      "num_tokens": 85985106.0,
      "reward": 0.447916679084301,
      "reward_std": 0.16413039481267333,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1478.8125610351562,
      "epoch": 0.18819011041766684,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.789349892925186e-07,
      "loss": 0.1267,
      "num_tokens": 86141888.0,
      "reward": 0.4791666716337204,
      "reward_std": 0.17640681471675634,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.0208740234375,
      "epoch": 0.1885741718674988,
      "grad_norm": 0.2119140625,
      "learning_rate": 9.78752078501744e-07,
      "loss": 0.0821,
      "num_tokens": 86306794.0,
      "reward": 0.533854179084301,
      "reward_std": 0.23055442050099373,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1852.0625305175781,
      "epoch": 0.18895823331733078,
      "grad_norm": 0.2001953125,
      "learning_rate": 9.78568396217708e-07,
      "loss": 0.0462,
      "num_tokens": 86497760.0,
      "reward": 0.361979179084301,
      "reward_std": 0.13113981671631336,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515624925494194,
      "step": 492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1875.4063110351562,
      "epoch": 0.18934229476716274,
      "grad_norm": 0.166015625,
      "learning_rate": 9.783839427709268e-07,
      "loss": 0.0384,
      "num_tokens": 86691831.0,
      "reward": 0.3906250074505806,
      "reward_std": 0.15047956071794033,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.7917175292969,
      "epoch": 0.1897263562169947,
      "grad_norm": 0.21875,
      "learning_rate": 9.781987184933044e-07,
      "loss": 0.0962,
      "num_tokens": 86859091.0,
      "reward": 0.5781250149011612,
      "reward_std": 0.24942708387970924,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.6250305175781,
      "epoch": 0.19011041766682668,
      "grad_norm": 0.228515625,
      "learning_rate": 9.780127237181322e-07,
      "loss": 0.0698,
      "num_tokens": 87025399.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.16012105345726013,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.2396240234375,
      "epoch": 0.19049447911665868,
      "grad_norm": 0.212890625,
      "learning_rate": 9.778259587800878e-07,
      "loss": 0.0733,
      "num_tokens": 87196022.0,
      "reward": 0.4869791939854622,
      "reward_std": 0.14467704575508833,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1869.6354370117188,
      "epoch": 0.19087854056649065,
      "grad_norm": 0.1591796875,
      "learning_rate": 9.776384240152347e-07,
      "loss": 0.0597,
      "num_tokens": 87391131.0,
      "reward": 0.455729179084301,
      "reward_std": 0.20728326216340065,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291641831398,
      "step": 497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1535.6771240234375,
      "epoch": 0.1912626020163226,
      "grad_norm": 0.216796875,
      "learning_rate": 9.774501197610214e-07,
      "loss": 0.0895,
      "num_tokens": 87552868.0,
      "reward": 0.5104166716337204,
      "reward_std": 0.1343256440013647,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.5000305175781,
      "epoch": 0.19164666346615458,
      "grad_norm": 0.240234375,
      "learning_rate": 9.77261046356281e-07,
      "loss": 0.1515,
      "num_tokens": 87724500.0,
      "reward": 0.4453125223517418,
      "reward_std": 0.16546273604035378,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828124925494194,
      "step": 499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.4479370117188,
      "epoch": 0.19203072491598655,
      "grad_norm": 0.181640625,
      "learning_rate": 9.770712041412312e-07,
      "loss": 0.0964,
      "num_tokens": 87919503.0,
      "reward": 0.416666679084301,
      "reward_std": 0.17317516636103392,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333283662796,
      "step": 500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1885.4166870117188,
      "epoch": 0.19241478636581852,
      "grad_norm": 0.16796875,
      "learning_rate": 9.768805934574723e-07,
      "loss": 0.0755,
      "num_tokens": 88114991.0,
      "reward": 0.3776041716337204,
      "reward_std": 0.1849935119971633,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375,
      "step": 501
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.9895935058594,
      "epoch": 0.19279884781565051,
      "grad_norm": 0.1708984375,
      "learning_rate": 9.76689214647988e-07,
      "loss": 0.0598,
      "num_tokens": 88294758.0,
      "reward": 0.401041679084301,
      "reward_std": 0.15288086980581284,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.6875305175781,
      "epoch": 0.19318290926548248,
      "grad_norm": 0.2138671875,
      "learning_rate": 9.764970680571436e-07,
      "loss": 0.0547,
      "num_tokens": 88465496.0,
      "reward": 0.463541679084301,
      "reward_std": 0.20556727424263954,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 503
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.9895935058594,
      "epoch": 0.19356697071531445,
      "grad_norm": 0.228515625,
      "learning_rate": 9.763041540306864e-07,
      "loss": 0.0934,
      "num_tokens": 88630751.0,
      "reward": 0.5234375223517418,
      "reward_std": 0.25894927605986595,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 504
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.0104675292969,
      "epoch": 0.19395103216514642,
      "grad_norm": 0.2333984375,
      "learning_rate": 9.761104729157444e-07,
      "loss": 0.0565,
      "num_tokens": 88796384.0,
      "reward": 0.5598958507180214,
      "reward_std": 0.1944404197856784,
      "rewards/accuracy_reward": 0.15625000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1421.6979370117188,
      "epoch": 0.1943350936149784,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.75916025060826e-07,
      "loss": 0.1567,
      "num_tokens": 88945499.0,
      "reward": 0.6328125298023224,
      "reward_std": 0.23138764686882496,
      "rewards/accuracy_reward": 0.19791667349636555,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.8646240234375,
      "epoch": 0.19471915506481036,
      "grad_norm": 0.205078125,
      "learning_rate": 9.757208108158189e-07,
      "loss": 0.0858,
      "num_tokens": 89111902.0,
      "reward": 0.4661458358168602,
      "reward_std": 0.2036086842417717,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 507
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.0313110351562,
      "epoch": 0.19510321651464235,
      "grad_norm": 0.212890625,
      "learning_rate": 9.755248305319904e-07,
      "loss": 0.0965,
      "num_tokens": 89281801.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.1504491474479437,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.5833740234375,
      "epoch": 0.19548727796447432,
      "grad_norm": 0.224609375,
      "learning_rate": 9.753280845619863e-07,
      "loss": 0.1639,
      "num_tokens": 89455529.0,
      "reward": 0.4687500223517418,
      "reward_std": 0.22520025819540024,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.4583740234375,
      "epoch": 0.1958713394143063,
      "grad_norm": 0.21484375,
      "learning_rate": 9.75130573259829e-07,
      "loss": 0.0844,
      "num_tokens": 89619381.0,
      "reward": 0.5494791865348816,
      "reward_std": 0.21106535382568836,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 510
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1559.3333740234375,
      "epoch": 0.19625540086413826,
      "grad_norm": 0.208984375,
      "learning_rate": 9.749322969809197e-07,
      "loss": 0.0642,
      "num_tokens": 89781885.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.23311161622405052,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1484.9687805175781,
      "epoch": 0.19663946231397023,
      "grad_norm": 0.255859375,
      "learning_rate": 9.747332560820346e-07,
      "loss": 0.1096,
      "num_tokens": 89937178.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.14983119256794453,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291567325592,
      "step": 512
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.5000305175781,
      "epoch": 0.19702352376380222,
      "grad_norm": 0.1572265625,
      "learning_rate": 9.745334509213268e-07,
      "loss": 0.0445,
      "num_tokens": 90117666.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.15091909933835268,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1567.1875305175781,
      "epoch": 0.1974075852136342,
      "grad_norm": 0.2099609375,
      "learning_rate": 9.74332881858324e-07,
      "loss": 0.1467,
      "num_tokens": 90282388.0,
      "reward": 0.5911458432674408,
      "reward_std": 0.27559157088398933,
      "rewards/accuracy_reward": 0.17708333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.5,
      "epoch": 0.19779164666346616,
      "grad_norm": 0.2109375,
      "learning_rate": 9.741315492539287e-07,
      "loss": 0.0844,
      "num_tokens": 90447428.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.227317338809371,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.5416870117188,
      "epoch": 0.19817570811329813,
      "grad_norm": 0.171875,
      "learning_rate": 9.739294534704174e-07,
      "loss": 0.0978,
      "num_tokens": 90620656.0,
      "reward": 0.4817708358168602,
      "reward_std": 0.23871491383761168,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 516
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.8958740234375,
      "epoch": 0.1985597695631301,
      "grad_norm": 0.197265625,
      "learning_rate": 9.737265948714394e-07,
      "loss": 0.1071,
      "num_tokens": 90789846.0,
      "reward": 0.5234375223517418,
      "reward_std": 0.11195855727419257,
      "rewards/accuracy_reward": 0.1458333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 517
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.9062805175781,
      "epoch": 0.19894383101296206,
      "grad_norm": 0.208984375,
      "learning_rate": 9.735229738220174e-07,
      "loss": 0.0758,
      "num_tokens": 90954837.0,
      "reward": 0.5833333507180214,
      "reward_std": 0.2141057439148426,
      "rewards/accuracy_reward": 0.16666667256504297,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 518
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1648.0208740234375,
      "epoch": 0.19932789246279406,
      "grad_norm": 0.220703125,
      "learning_rate": 9.73318590688545e-07,
      "loss": 0.1179,
      "num_tokens": 91128471.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.19154679775238037,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 519
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.2500305175781,
      "epoch": 0.19971195391262603,
      "grad_norm": 0.2060546875,
      "learning_rate": 9.731134458387884e-07,
      "loss": 0.1128,
      "num_tokens": 91306631.0,
      "reward": 0.4479166716337204,
      "reward_std": 0.17201154865324497,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 520
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1454.9479370117188,
      "epoch": 0.200096015362458,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.729075396418828e-07,
      "loss": 0.1166,
      "num_tokens": 91462010.0,
      "reward": 0.6250000149011612,
      "reward_std": 0.3255181759595871,
      "rewards/accuracy_reward": 0.16666666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 521
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.3333435058594,
      "epoch": 0.20048007681228996,
      "grad_norm": 0.1826171875,
      "learning_rate": 9.727008724683351e-07,
      "loss": 0.0776,
      "num_tokens": 91633938.0,
      "reward": 0.5234375223517418,
      "reward_std": 0.22142642177641392,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 522
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1482.6042175292969,
      "epoch": 0.20086413826212193,
      "grad_norm": 0.1826171875,
      "learning_rate": 9.724934446900203e-07,
      "loss": 0.0786,
      "num_tokens": 91789796.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.18134726397693157,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.3229370117188,
      "epoch": 0.2012481997119539,
      "grad_norm": 0.18359375,
      "learning_rate": 9.722852566801824e-07,
      "loss": 0.1262,
      "num_tokens": 91964475.0,
      "reward": 0.4635416865348816,
      "reward_std": 0.17893983516842127,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.6458740234375,
      "epoch": 0.2016322611617859,
      "grad_norm": 0.205078125,
      "learning_rate": 9.720763088134333e-07,
      "loss": 0.0968,
      "num_tokens": 92146793.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.20553742721676826,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1813.7396240234375,
      "epoch": 0.20201632261161787,
      "grad_norm": 0.19921875,
      "learning_rate": 9.718666014657521e-07,
      "loss": 0.0956,
      "num_tokens": 92335576.0,
      "reward": 0.3489583432674408,
      "reward_std": 0.12271495908498764,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583283662796,
      "step": 526
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.4479370117188,
      "epoch": 0.20240038406144983,
      "grad_norm": 0.23046875,
      "learning_rate": 9.716561350144849e-07,
      "loss": 0.1419,
      "num_tokens": 92508139.0,
      "reward": 0.4088541716337204,
      "reward_std": 0.15898765623569489,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 527
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.4688110351562,
      "epoch": 0.2027844455112818,
      "grad_norm": 0.208984375,
      "learning_rate": 9.714449098383433e-07,
      "loss": 0.1356,
      "num_tokens": 92678008.0,
      "reward": 0.5390625149011612,
      "reward_std": 0.1987907700240612,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 528
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.9479675292969,
      "epoch": 0.20316850696111377,
      "grad_norm": 0.1865234375,
      "learning_rate": 9.712329263174045e-07,
      "loss": 0.07,
      "num_tokens": 92863379.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.1223430847749114,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 529
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.5104370117188,
      "epoch": 0.20355256841094574,
      "grad_norm": 0.2412109375,
      "learning_rate": 9.710201848331097e-07,
      "loss": 0.0941,
      "num_tokens": 93019164.0,
      "reward": 0.5885416865348816,
      "reward_std": 0.2519732993096113,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583283662796,
      "step": 530
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.9167175292969,
      "epoch": 0.20393662986077774,
      "grad_norm": 0.2294921875,
      "learning_rate": 9.708066857682645e-07,
      "loss": 0.0817,
      "num_tokens": 93181804.0,
      "reward": 0.5312500223517418,
      "reward_std": 0.26132428646087646,
      "rewards/accuracy_reward": 0.12500000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 531
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.7917175292969,
      "epoch": 0.2043206913106097,
      "grad_norm": 0.1884765625,
      "learning_rate": 9.705924295070376e-07,
      "loss": 0.0673,
      "num_tokens": 93351584.0,
      "reward": 0.4375,
      "reward_std": 0.17022395320236683,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.5104370117188,
      "epoch": 0.20470475276044167,
      "grad_norm": 0.181640625,
      "learning_rate": 9.703774164349598e-07,
      "loss": 0.1011,
      "num_tokens": 93539609.0,
      "reward": 0.440104179084301,
      "reward_std": 0.13159554405137897,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 533
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1428.6979675292969,
      "epoch": 0.20508881421027364,
      "grad_norm": 0.2412109375,
      "learning_rate": 9.701616469389244e-07,
      "loss": 0.1046,
      "num_tokens": 93694164.0,
      "reward": 0.5885416865348816,
      "reward_std": 0.250649593770504,
      "rewards/accuracy_reward": 0.13541666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 534
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.6250305175781,
      "epoch": 0.2054728756601056,
      "grad_norm": 0.23046875,
      "learning_rate": 9.699451214071852e-07,
      "loss": 0.1343,
      "num_tokens": 93861208.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.23392893373966217,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.5729370117188,
      "epoch": 0.20585693710993758,
      "grad_norm": 0.177734375,
      "learning_rate": 9.697278402293566e-07,
      "loss": 0.0578,
      "num_tokens": 94028687.0,
      "reward": 0.4010416716337204,
      "reward_std": 0.11310964915901423,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.0104675292969,
      "epoch": 0.20624099855976957,
      "grad_norm": 0.2255859375,
      "learning_rate": 9.695098037964126e-07,
      "loss": 0.0464,
      "num_tokens": 94193992.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.21784158051013947,
      "rewards/accuracy_reward": 0.10416666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 537
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.0312805175781,
      "epoch": 0.20662506000960154,
      "grad_norm": 0.2392578125,
      "learning_rate": 9.692910125006865e-07,
      "loss": 0.1273,
      "num_tokens": 94369987.0,
      "reward": 0.502604179084301,
      "reward_std": 0.256554139778018,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 538
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.0104370117188,
      "epoch": 0.2070091214594335,
      "grad_norm": 0.173828125,
      "learning_rate": 9.690714667358696e-07,
      "loss": 0.0625,
      "num_tokens": 94556972.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.17158192675560713,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416641831398,
      "step": 539
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.9375305175781,
      "epoch": 0.20739318290926548,
      "grad_norm": 0.193359375,
      "learning_rate": 9.688511668970113e-07,
      "loss": 0.1178,
      "num_tokens": 94746086.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.22922010719776154,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.5729675292969,
      "epoch": 0.20777724435909745,
      "grad_norm": 0.205078125,
      "learning_rate": 9.686301133805168e-07,
      "loss": 0.118,
      "num_tokens": 94916733.0,
      "reward": 0.4661458507180214,
      "reward_std": 0.17641080543398857,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 541
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.7708435058594,
      "epoch": 0.20816130580892944,
      "grad_norm": 0.185546875,
      "learning_rate": 9.684083065841488e-07,
      "loss": 0.0959,
      "num_tokens": 95088815.0,
      "reward": 0.4713541865348816,
      "reward_std": 0.18188237119466066,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 542
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.5729370117188,
      "epoch": 0.2085453672587614,
      "grad_norm": 0.203125,
      "learning_rate": 9.681857469070241e-07,
      "loss": 0.1302,
      "num_tokens": 95250822.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.20566173642873764,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 543
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.1771240234375,
      "epoch": 0.20892942870859338,
      "grad_norm": 0.2255859375,
      "learning_rate": 9.679624347496155e-07,
      "loss": 0.0447,
      "num_tokens": 95431879.0,
      "reward": 0.416666679084301,
      "reward_std": 0.20375800505280495,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3749999925494194,
      "step": 544
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.2500305175781,
      "epoch": 0.20931349015842535,
      "grad_norm": 0.216796875,
      "learning_rate": 9.677383705137487e-07,
      "loss": 0.0959,
      "num_tokens": 95601607.0,
      "reward": 0.5833333507180214,
      "reward_std": 0.2521221172064543,
      "rewards/accuracy_reward": 0.15624999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 545
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.8021240234375,
      "epoch": 0.20969755160825732,
      "grad_norm": 0.24609375,
      "learning_rate": 9.675135546026037e-07,
      "loss": 0.0405,
      "num_tokens": 95772372.0,
      "reward": 0.510416679084301,
      "reward_std": 0.21659925393760204,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 546
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.0313110351562,
      "epoch": 0.21008161305808928,
      "grad_norm": 0.203125,
      "learning_rate": 9.672879874207123e-07,
      "loss": 0.101,
      "num_tokens": 95952167.0,
      "reward": 0.5781250298023224,
      "reward_std": 0.3114995248615742,
      "rewards/accuracy_reward": 0.18749999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 547
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1738.0104370117188,
      "epoch": 0.21046567450792128,
      "grad_norm": 0.216796875,
      "learning_rate": 9.670616693739583e-07,
      "loss": 0.0908,
      "num_tokens": 96133536.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.21281882375478745,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.6979370117188,
      "epoch": 0.21084973595775325,
      "grad_norm": 0.2412109375,
      "learning_rate": 9.668346008695769e-07,
      "loss": 0.0902,
      "num_tokens": 96297915.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.22641818597912788,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140624925494194,
      "step": 549
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.5000305175781,
      "epoch": 0.21123379740758522,
      "grad_norm": 0.1748046875,
      "learning_rate": 9.666067823161532e-07,
      "loss": 0.1004,
      "num_tokens": 96481723.0,
      "reward": 0.385416679084301,
      "reward_std": 0.11063354462385178,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333358168602,
      "step": 550
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.5417175292969,
      "epoch": 0.21161785885741718,
      "grad_norm": 0.2138671875,
      "learning_rate": 9.663782141236225e-07,
      "loss": 0.1271,
      "num_tokens": 96657647.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.17812114022672176,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 551
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.1145935058594,
      "epoch": 0.21200192030724915,
      "grad_norm": 0.2294921875,
      "learning_rate": 9.661488967032685e-07,
      "loss": 0.0556,
      "num_tokens": 96835626.0,
      "reward": 0.408854179084301,
      "reward_std": 0.13795156590640545,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 552
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.9271240234375,
      "epoch": 0.21238598175708112,
      "grad_norm": 0.1875,
      "learning_rate": 9.659188304677232e-07,
      "loss": 0.125,
      "num_tokens": 97011355.0,
      "reward": 0.4140625149011612,
      "reward_std": 0.17984341084957123,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.8333740234375,
      "epoch": 0.21277004320691312,
      "grad_norm": 0.1767578125,
      "learning_rate": 9.656880158309664e-07,
      "loss": 0.0264,
      "num_tokens": 97196523.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.10954404715448618,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 554
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.5833740234375,
      "epoch": 0.21315410465674509,
      "grad_norm": 0.2392578125,
      "learning_rate": 9.654564532083235e-07,
      "loss": 0.0739,
      "num_tokens": 97365115.0,
      "reward": 0.471354179084301,
      "reward_std": 0.22193703427910805,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1883.3958740234375,
      "epoch": 0.21353816610657705,
      "grad_norm": 0.15234375,
      "learning_rate": 9.652241430164673e-07,
      "loss": 0.0486,
      "num_tokens": 97561481.0,
      "reward": 0.3776041716337204,
      "reward_std": 0.13056496623903513,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 556
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1794.6875305175781,
      "epoch": 0.21392222755640902,
      "grad_norm": 0.1669921875,
      "learning_rate": 9.649910856734147e-07,
      "loss": 0.0875,
      "num_tokens": 97751747.0,
      "reward": 0.447916679084301,
      "reward_std": 0.21196742728352547,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.34375,
      "step": 557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1567.1146240234375,
      "epoch": 0.214306289006241,
      "grad_norm": 0.279296875,
      "learning_rate": 9.647572815985272e-07,
      "loss": 0.0731,
      "num_tokens": 97920774.0,
      "reward": 0.4557291716337204,
      "reward_std": 0.11706832610070705,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.3542175292969,
      "epoch": 0.21469035045607296,
      "grad_norm": 0.1923828125,
      "learning_rate": 9.645227312125102e-07,
      "loss": 0.0882,
      "num_tokens": 98102256.0,
      "reward": 0.4036458358168602,
      "reward_std": 0.16792886145412922,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625074505806,
      "step": 559
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.6146240234375,
      "epoch": 0.21507441190590496,
      "grad_norm": 0.2099609375,
      "learning_rate": 9.64287434937412e-07,
      "loss": 0.1283,
      "num_tokens": 98277851.0,
      "reward": 0.536458358168602,
      "reward_std": 0.2928648889064789,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.2396240234375,
      "epoch": 0.21545847335573692,
      "grad_norm": 0.228515625,
      "learning_rate": 9.640513931966228e-07,
      "loss": 0.086,
      "num_tokens": 98444274.0,
      "reward": 0.5286458432674408,
      "reward_std": 0.28782114014029503,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958432674408,
      "step": 561
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1726.8854675292969,
      "epoch": 0.2158425348055689,
      "grad_norm": 0.16796875,
      "learning_rate": 9.638146064148744e-07,
      "loss": 0.0486,
      "num_tokens": 98624423.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.14182595536112785,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 562
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1254.8542175292969,
      "epoch": 0.21622659625540086,
      "grad_norm": 0.265625,
      "learning_rate": 9.63577075018239e-07,
      "loss": 0.0815,
      "num_tokens": 98759745.0,
      "reward": 0.723958358168602,
      "reward_std": 0.4311072826385498,
      "rewards/accuracy_reward": 0.17708333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.546875,
      "step": 563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.3958740234375,
      "epoch": 0.21661065770523283,
      "grad_norm": 0.1884765625,
      "learning_rate": 9.633387994341294e-07,
      "loss": 0.1367,
      "num_tokens": 98933623.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.18513481877744198,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 564
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.5208740234375,
      "epoch": 0.2169947191550648,
      "grad_norm": 0.193359375,
      "learning_rate": 9.630997800912962e-07,
      "loss": 0.0384,
      "num_tokens": 99111537.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.09825327526777983,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 565
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.3646240234375,
      "epoch": 0.2173787806048968,
      "grad_norm": 0.21484375,
      "learning_rate": 9.62860017419829e-07,
      "loss": 0.0783,
      "num_tokens": 99285524.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.11250572372227907,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1818.7917175292969,
      "epoch": 0.21776284205472876,
      "grad_norm": 0.2041015625,
      "learning_rate": 9.626195118511555e-07,
      "loss": 0.1133,
      "num_tokens": 99475648.0,
      "reward": 0.3697916716337204,
      "reward_std": 0.1314333062618971,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 567
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.1458740234375,
      "epoch": 0.21814690350456073,
      "grad_norm": 0.2236328125,
      "learning_rate": 9.623782638180391e-07,
      "loss": 0.0843,
      "num_tokens": 99661598.0,
      "reward": 0.4114583358168602,
      "reward_std": 0.19094307534396648,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 568
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.7812805175781,
      "epoch": 0.2185309649543927,
      "grad_norm": 0.2197265625,
      "learning_rate": 9.6213627375458e-07,
      "loss": 0.0397,
      "num_tokens": 99843761.0,
      "reward": 0.479166679084301,
      "reward_std": 0.23570828512310982,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3750000074505806,
      "step": 569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1667.1979675292969,
      "epoch": 0.21891502640422467,
      "grad_norm": 0.220703125,
      "learning_rate": 9.618935420962131e-07,
      "loss": 0.0493,
      "num_tokens": 100018124.0,
      "reward": 0.419270858168602,
      "reward_std": 0.1688994150608778,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 570
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.7500610351562,
      "epoch": 0.21929908785405666,
      "grad_norm": 0.189453125,
      "learning_rate": 9.616500692797081e-07,
      "loss": 0.1211,
      "num_tokens": 100189404.0,
      "reward": 0.432291679084301,
      "reward_std": 0.17321653477847576,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 571
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.5833435058594,
      "epoch": 0.21968314930388863,
      "grad_norm": 0.244140625,
      "learning_rate": 9.61405855743168e-07,
      "loss": 0.121,
      "num_tokens": 100350932.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.2251361943781376,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.3854675292969,
      "epoch": 0.2200672107537206,
      "grad_norm": 0.1572265625,
      "learning_rate": 9.611609019260289e-07,
      "loss": 0.0543,
      "num_tokens": 100537977.0,
      "reward": 0.3489583432674408,
      "reward_std": 0.10470135603100061,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416716337204,
      "step": 573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.947998046875,
      "epoch": 0.22045127220355257,
      "grad_norm": 0.23828125,
      "learning_rate": 9.609152082690588e-07,
      "loss": 0.1298,
      "num_tokens": 100697468.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.24080885015428066,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 574
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.4375305175781,
      "epoch": 0.22083533365338454,
      "grad_norm": 0.2021484375,
      "learning_rate": 9.60668775214357e-07,
      "loss": 0.0997,
      "num_tokens": 100865566.0,
      "reward": 0.533854179084301,
      "reward_std": 0.28606490045785904,
      "rewards/accuracy_reward": 0.10416666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875074505806,
      "step": 575
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.0521545410156,
      "epoch": 0.2212193951032165,
      "grad_norm": 0.216796875,
      "learning_rate": 9.604216032053535e-07,
      "loss": 0.0921,
      "num_tokens": 101032059.0,
      "reward": 0.4713541939854622,
      "reward_std": 0.22954711318016052,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.8021240234375,
      "epoch": 0.2216034565530485,
      "grad_norm": 0.197265625,
      "learning_rate": 9.601736926868073e-07,
      "loss": 0.0793,
      "num_tokens": 101210440.0,
      "reward": 0.440104179084301,
      "reward_std": 0.1579340510070324,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.3958740234375,
      "epoch": 0.22198751800288047,
      "grad_norm": 0.2255859375,
      "learning_rate": 9.599250441048073e-07,
      "loss": 0.0712,
      "num_tokens": 101379934.0,
      "reward": 0.5078125223517418,
      "reward_std": 0.21284744143486023,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140624925494194,
      "step": 578
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.1979370117188,
      "epoch": 0.22237157945271244,
      "grad_norm": 0.2265625,
      "learning_rate": 9.59675657906769e-07,
      "loss": 0.0893,
      "num_tokens": 101555953.0,
      "reward": 0.4114583358168602,
      "reward_std": 0.12199891079217196,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 579
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1496.9688110351562,
      "epoch": 0.2227556409025444,
      "grad_norm": 0.2041015625,
      "learning_rate": 9.594255345414372e-07,
      "loss": 0.0718,
      "num_tokens": 101712150.0,
      "reward": 0.6536458507180214,
      "reward_std": 0.26995671540498734,
      "rewards/accuracy_reward": 0.20833333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 580
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.3750305175781,
      "epoch": 0.22313970235237637,
      "grad_norm": 0.1728515625,
      "learning_rate": 9.591746744588813e-07,
      "loss": 0.0602,
      "num_tokens": 101896626.0,
      "reward": 0.3984375074505806,
      "reward_std": 0.12364847538992763,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.4583740234375,
      "epoch": 0.22352376380220834,
      "grad_norm": 0.21484375,
      "learning_rate": 9.58923078110497e-07,
      "loss": 0.1455,
      "num_tokens": 102056502.0,
      "reward": 0.5208333507180214,
      "reward_std": 0.24808101635426283,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 582
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.7709045410156,
      "epoch": 0.22390782525204034,
      "grad_norm": 0.1640625,
      "learning_rate": 9.586707459490052e-07,
      "loss": 0.0379,
      "num_tokens": 102243400.0,
      "reward": 0.4843750074505806,
      "reward_std": 0.18940653465688229,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583432674408,
      "step": 583
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.3334045410156,
      "epoch": 0.2242918867018723,
      "grad_norm": 0.21484375,
      "learning_rate": 9.584176784284504e-07,
      "loss": 0.0624,
      "num_tokens": 102425056.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.21453658863902092,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.5208740234375,
      "epoch": 0.22467594815170427,
      "grad_norm": 0.2578125,
      "learning_rate": 9.581638760042e-07,
      "loss": 0.0383,
      "num_tokens": 102582666.0,
      "reward": 0.572916679084301,
      "reward_std": 0.28373170644044876,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833432674408,
      "step": 585
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.9062805175781,
      "epoch": 0.22506000960153624,
      "grad_norm": 0.205078125,
      "learning_rate": 9.579093391329449e-07,
      "loss": 0.0317,
      "num_tokens": 102752513.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.2131177056580782,
      "rewards/accuracy_reward": 0.10937500093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 586
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.3750305175781,
      "epoch": 0.2254440710513682,
      "grad_norm": 0.21484375,
      "learning_rate": 9.576540682726963e-07,
      "loss": 0.0841,
      "num_tokens": 102928693.0,
      "reward": 0.3802083358168602,
      "reward_std": 0.11906595062464476,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 587
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.0312805175781,
      "epoch": 0.22582813250120018,
      "grad_norm": 0.259765625,
      "learning_rate": 9.573980638827866e-07,
      "loss": 0.083,
      "num_tokens": 103101280.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.1988964006304741,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 588
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1835.2396240234375,
      "epoch": 0.22621219395103218,
      "grad_norm": 0.1689453125,
      "learning_rate": 9.571413264238685e-07,
      "loss": 0.0659,
      "num_tokens": 103289159.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.21353046782314777,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.9687805175781,
      "epoch": 0.22659625540086414,
      "grad_norm": 0.1845703125,
      "learning_rate": 9.568838563579133e-07,
      "loss": 0.09,
      "num_tokens": 103472140.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.15419000200927258,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 590
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.6562805175781,
      "epoch": 0.2269803168506961,
      "grad_norm": 0.2109375,
      "learning_rate": 9.566256541482106e-07,
      "loss": 0.1256,
      "num_tokens": 103647763.0,
      "reward": 0.5000000074505806,
      "reward_std": 0.24296747893095016,
      "rewards/accuracy_reward": 0.11458333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 591
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.7604675292969,
      "epoch": 0.22736437830052808,
      "grad_norm": 0.2216796875,
      "learning_rate": 9.563667202593677e-07,
      "loss": 0.1309,
      "num_tokens": 103838156.0,
      "reward": 0.4531250223517418,
      "reward_std": 0.19611174520105124,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.6563110351562,
      "epoch": 0.22774843975036005,
      "grad_norm": 0.1982421875,
      "learning_rate": 9.561070551573081e-07,
      "loss": 0.1033,
      "num_tokens": 103996739.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.16257354244589806,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.7187805175781,
      "epoch": 0.22813250120019202,
      "grad_norm": 0.1728515625,
      "learning_rate": 9.558466593092714e-07,
      "loss": 0.055,
      "num_tokens": 104169504.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.20175987482070923,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.7813110351562,
      "epoch": 0.228516562650024,
      "grad_norm": 0.1923828125,
      "learning_rate": 9.55585533183812e-07,
      "loss": 0.0536,
      "num_tokens": 104329467.0,
      "reward": 0.5104166939854622,
      "reward_std": 0.15547055657953024,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 595
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.5417175292969,
      "epoch": 0.22890062409985598,
      "grad_norm": 0.1953125,
      "learning_rate": 9.553236772507982e-07,
      "loss": 0.1408,
      "num_tokens": 104504983.0,
      "reward": 0.4609375223517418,
      "reward_std": 0.2524569220840931,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 596
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.9166870117188,
      "epoch": 0.22928468554968795,
      "grad_norm": 0.17578125,
      "learning_rate": 9.55061091981411e-07,
      "loss": 0.0872,
      "num_tokens": 104684063.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.187366864643991,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 597
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.0000610351562,
      "epoch": 0.22966874699951992,
      "grad_norm": 0.1640625,
      "learning_rate": 9.547977778481456e-07,
      "loss": 0.0574,
      "num_tokens": 104873271.0,
      "reward": 0.4687500074505806,
      "reward_std": 0.20277253910899162,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 598
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.8646545410156,
      "epoch": 0.2300528084493519,
      "grad_norm": 0.1533203125,
      "learning_rate": 9.545337353248064e-07,
      "loss": 0.0687,
      "num_tokens": 105053098.0,
      "reward": 0.385416679084301,
      "reward_std": 0.10819746926426888,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.5104370117188,
      "epoch": 0.23043686989918388,
      "grad_norm": 0.2158203125,
      "learning_rate": 9.5426896488651e-07,
      "loss": 0.0825,
      "num_tokens": 105235771.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.13552747666835785,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.7708740234375,
      "epoch": 0.23082093134901585,
      "grad_norm": 0.2001953125,
      "learning_rate": 9.54003467009682e-07,
      "loss": 0.1152,
      "num_tokens": 105413277.0,
      "reward": 0.479166679084301,
      "reward_std": 0.26211462542414665,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 601
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.9583740234375,
      "epoch": 0.23120499279884782,
      "grad_norm": 0.2431640625,
      "learning_rate": 9.537372421720579e-07,
      "loss": 0.1268,
      "num_tokens": 105597105.0,
      "reward": 0.3619791716337204,
      "reward_std": 0.09582891641184688,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 602
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.0000305175781,
      "epoch": 0.2315890542486798,
      "grad_norm": 0.1982421875,
      "learning_rate": 9.534702908526801e-07,
      "loss": 0.106,
      "num_tokens": 105782273.0,
      "reward": 0.4088541865348816,
      "reward_std": 0.1534388829022646,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 603
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1374.9166870117188,
      "epoch": 0.23197311569851176,
      "grad_norm": 0.25,
      "learning_rate": 9.532026135318988e-07,
      "loss": 0.1762,
      "num_tokens": 105927041.0,
      "reward": 0.5677083358168602,
      "reward_std": 0.3117874227464199,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 604
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1357.6041870117188,
      "epoch": 0.23235717714834372,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.52934210691371e-07,
      "loss": 0.1224,
      "num_tokens": 106070411.0,
      "reward": 0.622395858168602,
      "reward_std": 0.3269096836447716,
      "rewards/accuracy_reward": 0.15625000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458283662796,
      "step": 605
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.8021240234375,
      "epoch": 0.23274123859817572,
      "grad_norm": 0.1943359375,
      "learning_rate": 9.526650828140583e-07,
      "loss": 0.0884,
      "num_tokens": 106226576.0,
      "reward": 0.486979179084301,
      "reward_std": 0.18468278273940086,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 606
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.0729675292969,
      "epoch": 0.2331253000480077,
      "grad_norm": 0.193359375,
      "learning_rate": 9.523952303842276e-07,
      "loss": 0.122,
      "num_tokens": 106405247.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.2539554536342621,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 607
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.4271545410156,
      "epoch": 0.23350936149783966,
      "grad_norm": 0.177734375,
      "learning_rate": 9.521246538874494e-07,
      "loss": 0.0776,
      "num_tokens": 106581792.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.09808994550257921,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 608
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.9896240234375,
      "epoch": 0.23389342294767163,
      "grad_norm": 0.185546875,
      "learning_rate": 9.518533538105967e-07,
      "loss": 0.0738,
      "num_tokens": 106764255.0,
      "reward": 0.3437500074505806,
      "reward_std": 0.08183515956625342,
      "rewards/accuracy_reward": 0.015625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333283662796,
      "step": 609
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.3125610351562,
      "epoch": 0.2342774843975036,
      "grad_norm": 0.171875,
      "learning_rate": 9.515813306418452e-07,
      "loss": 0.0474,
      "num_tokens": 106951069.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.23772989958524704,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 610
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.6979675292969,
      "epoch": 0.23466154584733556,
      "grad_norm": 0.2412109375,
      "learning_rate": 9.51308584870671e-07,
      "loss": 0.0289,
      "num_tokens": 107119280.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.2580725196748972,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 611
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.3645935058594,
      "epoch": 0.23504560729716756,
      "grad_norm": 0.201171875,
      "learning_rate": 9.510351169878511e-07,
      "loss": 0.0843,
      "num_tokens": 107274411.0,
      "reward": 0.5651041865348816,
      "reward_std": 0.2593704052269459,
      "rewards/accuracy_reward": 0.12500000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041567325592,
      "step": 612
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.9271240234375,
      "epoch": 0.23542966874699953,
      "grad_norm": 0.2109375,
      "learning_rate": 9.507609274854613e-07,
      "loss": 0.0912,
      "num_tokens": 107443340.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.13166073337197304,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 613
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.0313110351562,
      "epoch": 0.2358137301968315,
      "grad_norm": 0.205078125,
      "learning_rate": 9.504860168568762e-07,
      "loss": 0.101,
      "num_tokens": 107605127.0,
      "reward": 0.5078125223517418,
      "reward_std": 0.23150451388210058,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 614
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1333.3750457763672,
      "epoch": 0.23619779164666346,
      "grad_norm": 0.2578125,
      "learning_rate": 9.502103855967679e-07,
      "loss": 0.1024,
      "num_tokens": 107749371.0,
      "reward": 0.6354166865348816,
      "reward_std": 0.3299306780099869,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5,
      "step": 615
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.6041870117188,
      "epoch": 0.23658185309649543,
      "grad_norm": 0.21875,
      "learning_rate": 9.499340342011053e-07,
      "loss": 0.1272,
      "num_tokens": 107925797.0,
      "reward": 0.4583333358168602,
      "reward_std": 0.24446123652160168,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 616
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.7708740234375,
      "epoch": 0.2369659145463274,
      "grad_norm": 0.208984375,
      "learning_rate": 9.496569631671529e-07,
      "loss": 0.0956,
      "num_tokens": 108103111.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.2087236624211073,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 617
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.1875305175781,
      "epoch": 0.2373499759961594,
      "grad_norm": 0.2021484375,
      "learning_rate": 9.493791729934703e-07,
      "loss": 0.1248,
      "num_tokens": 108284873.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.20203788578510284,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 618
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.6042175292969,
      "epoch": 0.23773403744599136,
      "grad_norm": 0.2177734375,
      "learning_rate": 9.491006641799112e-07,
      "loss": 0.078,
      "num_tokens": 108465243.0,
      "reward": 0.494791679084301,
      "reward_std": 0.20421494916081429,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 619
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.1041870117188,
      "epoch": 0.23811809889582333,
      "grad_norm": 0.193359375,
      "learning_rate": 9.48821437227622e-07,
      "loss": 0.0874,
      "num_tokens": 108628045.0,
      "reward": 0.5546875149011612,
      "reward_std": 0.22710319561883807,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 620
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.5208435058594,
      "epoch": 0.2385021603456553,
      "grad_norm": 0.1962890625,
      "learning_rate": 9.485414926390417e-07,
      "loss": 0.1293,
      "num_tokens": 108814711.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.22795888036489487,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875074505806,
      "step": 621
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.2291870117188,
      "epoch": 0.23888622179548727,
      "grad_norm": 0.1923828125,
      "learning_rate": 9.482608309179005e-07,
      "loss": 0.098,
      "num_tokens": 109006813.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.19174083322286606,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 622
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1471.5208740234375,
      "epoch": 0.23927028324531924,
      "grad_norm": 0.2890625,
      "learning_rate": 9.479794525692187e-07,
      "loss": 0.0762,
      "num_tokens": 109159423.0,
      "reward": 0.6197916865348816,
      "reward_std": 0.21823174692690372,
      "rewards/accuracy_reward": 0.1770833283662796,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 623
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.0833740234375,
      "epoch": 0.23965434469515123,
      "grad_norm": 0.2333984375,
      "learning_rate": 9.47697358099307e-07,
      "loss": 0.0948,
      "num_tokens": 109318143.0,
      "reward": 0.5807291865348816,
      "reward_std": 0.2661703787744045,
      "rewards/accuracy_reward": 0.15624999720603228,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 624
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.5834045410156,
      "epoch": 0.2400384061449832,
      "grad_norm": 0.208984375,
      "learning_rate": 9.474145480157636e-07,
      "loss": 0.1095,
      "num_tokens": 109491575.0,
      "reward": 0.440104179084301,
      "reward_std": 0.17543018236756325,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 625
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.5521240234375,
      "epoch": 0.24042246759481517,
      "grad_norm": 0.1875,
      "learning_rate": 9.471310228274748e-07,
      "loss": 0.117,
      "num_tokens": 109673876.0,
      "reward": 0.3723958432674408,
      "reward_std": 0.14382007159292698,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 626
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.2500305175781,
      "epoch": 0.24080652904464714,
      "grad_norm": 0.1494140625,
      "learning_rate": 9.468467830446138e-07,
      "loss": 0.0504,
      "num_tokens": 109847108.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.22399595566093922,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.403645820915699,
      "step": 627
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.9896545410156,
      "epoch": 0.2411905904944791,
      "grad_norm": 0.185546875,
      "learning_rate": 9.465618291786397e-07,
      "loss": 0.0619,
      "num_tokens": 110037611.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.23290428146719933,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 628
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.6250305175781,
      "epoch": 0.2415746519443111,
      "grad_norm": 0.2412109375,
      "learning_rate": 9.46276161742296e-07,
      "loss": 0.1275,
      "num_tokens": 110204031.0,
      "reward": 0.5260416716337204,
      "reward_std": 0.2571035884320736,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 629
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.1875305175781,
      "epoch": 0.24195871339414307,
      "grad_norm": 0.2001953125,
      "learning_rate": 9.45989781249611e-07,
      "loss": 0.1454,
      "num_tokens": 110378441.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.2243056520819664,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958432674408,
      "step": 630
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1401.2187805175781,
      "epoch": 0.24234277484397504,
      "grad_norm": 0.28125,
      "learning_rate": 9.457026882158953e-07,
      "loss": 0.1263,
      "num_tokens": 110526054.0,
      "reward": 0.5104166716337204,
      "reward_std": 0.1970603819936514,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 631
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.2604370117188,
      "epoch": 0.242726836293807,
      "grad_norm": 0.23046875,
      "learning_rate": 9.454148831577418e-07,
      "loss": 0.0497,
      "num_tokens": 110683575.0,
      "reward": 0.5234375074505806,
      "reward_std": 0.1915721921250224,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541641831398,
      "step": 632
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.9792175292969,
      "epoch": 0.24311089774363898,
      "grad_norm": 0.208984375,
      "learning_rate": 9.451263665930249e-07,
      "loss": 0.0725,
      "num_tokens": 110858293.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.20668434351682663,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 633
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.9583740234375,
      "epoch": 0.24349495919347094,
      "grad_norm": 0.2470703125,
      "learning_rate": 9.448371390408993e-07,
      "loss": 0.0764,
      "num_tokens": 111015617.0,
      "reward": 0.643229179084301,
      "reward_std": 0.2954971678555012,
      "rewards/accuracy_reward": 0.20833333861082792,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 634
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.4167175292969,
      "epoch": 0.24387902064330294,
      "grad_norm": 0.1884765625,
      "learning_rate": 9.445472010217986e-07,
      "loss": 0.0454,
      "num_tokens": 111185201.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.16243898309767246,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.1458740234375,
      "epoch": 0.2442630820931349,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.442565530574353e-07,
      "loss": 0.0923,
      "num_tokens": 111356559.0,
      "reward": 0.5364583432674408,
      "reward_std": 0.254703595302999,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 636
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.1771240234375,
      "epoch": 0.24464714354296688,
      "grad_norm": 0.203125,
      "learning_rate": 9.439651956707991e-07,
      "loss": 0.0988,
      "num_tokens": 111531064.0,
      "reward": 0.4921875074505806,
      "reward_std": 0.22344615124166012,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 637
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1927.8437805175781,
      "epoch": 0.24503120499279885,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.436731293861563e-07,
      "loss": 0.0734,
      "num_tokens": 111729657.0,
      "reward": 0.3619791716337204,
      "reward_std": 0.18003366142511368,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291641831398,
      "step": 638
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1346.3958587646484,
      "epoch": 0.24541526644263081,
      "grad_norm": 0.2041015625,
      "learning_rate": 9.433803547290487e-07,
      "loss": 0.0717,
      "num_tokens": 111872087.0,
      "reward": 0.6328125149011612,
      "reward_std": 0.24603979662060738,
      "rewards/accuracy_reward": 0.18750000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 639
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.8854370117188,
      "epoch": 0.24579932789246278,
      "grad_norm": 0.181640625,
      "learning_rate": 9.43086872226293e-07,
      "loss": 0.0556,
      "num_tokens": 112051828.0,
      "reward": 0.4505208507180214,
      "reward_std": 0.12320363521575928,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 640
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1412.8333740234375,
      "epoch": 0.24618338934229478,
      "grad_norm": 0.2412109375,
      "learning_rate": 9.42792682405979e-07,
      "loss": 0.1052,
      "num_tokens": 112201636.0,
      "reward": 0.6562500223517418,
      "reward_std": 0.28590838238596916,
      "rewards/accuracy_reward": 0.19791666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333432674408,
      "step": 641
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1883.3438110351562,
      "epoch": 0.24656745079212675,
      "grad_norm": 0.169921875,
      "learning_rate": 9.424977857974701e-07,
      "loss": 0.0925,
      "num_tokens": 112396173.0,
      "reward": 0.3307291716337204,
      "reward_std": 0.1065320735797286,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291716337204,
      "step": 642
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1914.5833740234375,
      "epoch": 0.24695151224195871,
      "grad_norm": 0.140625,
      "learning_rate": 9.422021829314009e-07,
      "loss": 0.0866,
      "num_tokens": 112593485.0,
      "reward": 0.3281250074505806,
      "reward_std": 0.12341415137052536,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3177083432674408,
      "step": 643
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.9792175292969,
      "epoch": 0.24733557369179068,
      "grad_norm": 0.263671875,
      "learning_rate": 9.419058743396767e-07,
      "loss": 0.1273,
      "num_tokens": 112758819.0,
      "reward": 0.4635416716337204,
      "reward_std": 0.1418243944644928,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 644
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.3958587646484,
      "epoch": 0.24771963514162265,
      "grad_norm": 0.2373046875,
      "learning_rate": 9.416088605554733e-07,
      "loss": 0.0466,
      "num_tokens": 112923609.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.24995839782059193,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 645
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.0625305175781,
      "epoch": 0.24810369659145462,
      "grad_norm": 0.2001953125,
      "learning_rate": 9.413111421132349e-07,
      "loss": 0.1177,
      "num_tokens": 113090303.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.19204763509333134,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 646
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.3229370117188,
      "epoch": 0.24848775804128662,
      "grad_norm": 0.234375,
      "learning_rate": 9.410127195486739e-07,
      "loss": 0.0812,
      "num_tokens": 113253430.0,
      "reward": 0.565104179084301,
      "reward_std": 0.22146478667855263,
      "rewards/accuracy_reward": 0.1666666716337204,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 647
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.2292175292969,
      "epoch": 0.24887181949111858,
      "grad_norm": 0.20703125,
      "learning_rate": 9.407135933987696e-07,
      "loss": 0.1,
      "num_tokens": 113424900.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.2606325037777424,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 648
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.322998046875,
      "epoch": 0.24925588094095055,
      "grad_norm": 0.21875,
      "learning_rate": 9.404137642017673e-07,
      "loss": 0.1856,
      "num_tokens": 113596979.0,
      "reward": 0.4192708358168602,
      "reward_std": 0.18290542252361774,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 649
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.8542175292969,
      "epoch": 0.24963994239078252,
      "grad_norm": 0.208984375,
      "learning_rate": 9.401132324971774e-07,
      "loss": 0.1249,
      "num_tokens": 113769981.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.16187806241214275,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828124925494194,
      "step": 650
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.7083740234375,
      "epoch": 0.2500240038406145,
      "grad_norm": 0.216796875,
      "learning_rate": 9.398119988257743e-07,
      "loss": 0.0678,
      "num_tokens": 113938897.0,
      "reward": 0.4973958358168602,
      "reward_std": 0.19574039801955223,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 651
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.3333740234375,
      "epoch": 0.25040806529044646,
      "grad_norm": 0.234375,
      "learning_rate": 9.395100637295959e-07,
      "loss": 0.0885,
      "num_tokens": 114104113.0,
      "reward": 0.5104166716337204,
      "reward_std": 0.22400006465613842,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 652
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1809.2396240234375,
      "epoch": 0.25079212674027845,
      "grad_norm": 0.1796875,
      "learning_rate": 9.392074277519416e-07,
      "loss": 0.0797,
      "num_tokens": 114293224.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.17899328656494617,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.7500305175781,
      "epoch": 0.2511761881901104,
      "grad_norm": 0.1630859375,
      "learning_rate": 9.389040914373725e-07,
      "loss": -0.0102,
      "num_tokens": 114468568.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.14659635722637177,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 654
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.9479370117188,
      "epoch": 0.2515602496399424,
      "grad_norm": 0.1923828125,
      "learning_rate": 9.386000553317096e-07,
      "loss": 0.1051,
      "num_tokens": 114654067.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.23273301497101784,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 655
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.9688110351562,
      "epoch": 0.2519443110897744,
      "grad_norm": 0.20703125,
      "learning_rate": 9.382953199820331e-07,
      "loss": 0.1124,
      "num_tokens": 114823440.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.18624134128913283,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 656
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1677.8437805175781,
      "epoch": 0.2523283725396063,
      "grad_norm": 0.228515625,
      "learning_rate": 9.379898859366819e-07,
      "loss": 0.1099,
      "num_tokens": 114999273.0,
      "reward": 0.4557291716337204,
      "reward_std": 0.20885270833969116,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 657
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.5834045410156,
      "epoch": 0.2527124339894383,
      "grad_norm": 0.203125,
      "learning_rate": 9.37683753745251e-07,
      "loss": 0.1228,
      "num_tokens": 115178457.0,
      "reward": 0.4166666716337204,
      "reward_std": 0.18677676282823086,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 658
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1677.3437805175781,
      "epoch": 0.25309649543927026,
      "grad_norm": 0.2255859375,
      "learning_rate": 9.373769239585931e-07,
      "loss": 0.1232,
      "num_tokens": 115353066.0,
      "reward": 0.544270858168602,
      "reward_std": 0.33438868820667267,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 659
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.8333740234375,
      "epoch": 0.25348055688910226,
      "grad_norm": 0.220703125,
      "learning_rate": 9.370693971288149e-07,
      "loss": 0.1118,
      "num_tokens": 115535354.0,
      "reward": 0.4453125223517418,
      "reward_std": 0.21671806648373604,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 660
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.5833740234375,
      "epoch": 0.25386461833893426,
      "grad_norm": 0.26171875,
      "learning_rate": 9.367611738092781e-07,
      "loss": 0.1083,
      "num_tokens": 115698426.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.2405926138162613,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 661
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.2604675292969,
      "epoch": 0.2542486797887662,
      "grad_norm": 0.134765625,
      "learning_rate": 9.364522545545974e-07,
      "loss": 0.0418,
      "num_tokens": 115886371.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.11766774114221334,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 662
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.9792175292969,
      "epoch": 0.2546327412385982,
      "grad_norm": 0.1953125,
      "learning_rate": 9.361426399206398e-07,
      "loss": 0.0522,
      "num_tokens": 116068881.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.2035915134474635,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 663
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1476.1250305175781,
      "epoch": 0.25501680268843013,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.358323304645233e-07,
      "loss": 0.049,
      "num_tokens": 116224261.0,
      "reward": 0.5755208432674408,
      "reward_std": 0.21835422469303012,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4817708358168602,
      "step": 664
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.3438110351562,
      "epoch": 0.25540086413826213,
      "grad_norm": 0.234375,
      "learning_rate": 9.355213267446169e-07,
      "loss": 0.1146,
      "num_tokens": 116405654.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.17793676629662514,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1851.0417175292969,
      "epoch": 0.25578492558809407,
      "grad_norm": 0.2021484375,
      "learning_rate": 9.352096293205378e-07,
      "loss": 0.1007,
      "num_tokens": 116597738.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.1847998984158039,
      "rewards/accuracy_reward": 0.046875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 666
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.6458740234375,
      "epoch": 0.25616898703792607,
      "grad_norm": 0.2109375,
      "learning_rate": 9.348972387531522e-07,
      "loss": 0.1077,
      "num_tokens": 116779608.0,
      "reward": 0.421875,
      "reward_std": 0.22836144641041756,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 667
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.2396240234375,
      "epoch": 0.25655304848775806,
      "grad_norm": 0.2021484375,
      "learning_rate": 9.345841556045735e-07,
      "loss": 0.1113,
      "num_tokens": 116961359.0,
      "reward": 0.4088541716337204,
      "reward_std": 0.1294244583696127,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 668
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.5937805175781,
      "epoch": 0.25693710993759,
      "grad_norm": 0.2041015625,
      "learning_rate": 9.34270380438161e-07,
      "loss": 0.084,
      "num_tokens": 117154552.0,
      "reward": 0.3515625074505806,
      "reward_std": 0.14297222159802914,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 669
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1514.8437805175781,
      "epoch": 0.257321171387422,
      "grad_norm": 0.2236328125,
      "learning_rate": 9.339559138185193e-07,
      "loss": 0.0968,
      "num_tokens": 117313265.0,
      "reward": 0.5468750149011612,
      "reward_std": 0.26339541375637054,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916567325592,
      "step": 670
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.9479370117188,
      "epoch": 0.25770523283725394,
      "grad_norm": 0.205078125,
      "learning_rate": 9.336407563114975e-07,
      "loss": 0.0672,
      "num_tokens": 117481868.0,
      "reward": 0.432291679084301,
      "reward_std": 0.17980370158329606,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 671
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1892.9375305175781,
      "epoch": 0.25808929428708594,
      "grad_norm": 0.16796875,
      "learning_rate": 9.333249084841874e-07,
      "loss": 0.0564,
      "num_tokens": 117682150.0,
      "reward": 0.3932291865348816,
      "reward_std": 0.1493939971551299,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3203125074505806,
      "step": 672
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.0209045410156,
      "epoch": 0.25847335573691793,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.330083709049234e-07,
      "loss": 0.1506,
      "num_tokens": 117859952.0,
      "reward": 0.4166666716337204,
      "reward_std": 0.18201571702957153,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 673
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.0000305175781,
      "epoch": 0.25885741718674987,
      "grad_norm": 0.22265625,
      "learning_rate": 9.326911441432807e-07,
      "loss": 0.1111,
      "num_tokens": 118027848.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.20440133661031723,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 674
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.5208740234375,
      "epoch": 0.25924147863658187,
      "grad_norm": 0.21484375,
      "learning_rate": 9.323732287700749e-07,
      "loss": 0.055,
      "num_tokens": 118207522.0,
      "reward": 0.4661458507180214,
      "reward_std": 0.25384435430169106,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 675
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.5208740234375,
      "epoch": 0.2596255400864138,
      "grad_norm": 0.2080078125,
      "learning_rate": 9.320546253573603e-07,
      "loss": 0.103,
      "num_tokens": 118384772.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.20254699094220996,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 676
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.7396545410156,
      "epoch": 0.2600096015362458,
      "grad_norm": 0.2421875,
      "learning_rate": 9.317353344784298e-07,
      "loss": 0.1209,
      "num_tokens": 118553227.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.21904680645093322,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 677
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1869.1563110351562,
      "epoch": 0.26039366298607775,
      "grad_norm": 0.1689453125,
      "learning_rate": 9.314153567078126e-07,
      "loss": 0.0728,
      "num_tokens": 118746738.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.19462193362414837,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958432674408,
      "step": 678
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.1979370117188,
      "epoch": 0.26077772443590974,
      "grad_norm": 0.2177734375,
      "learning_rate": 9.310946926212746e-07,
      "loss": 0.1006,
      "num_tokens": 118917877.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.16645099874585867,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 679
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.2604675292969,
      "epoch": 0.26116178588574174,
      "grad_norm": 0.21484375,
      "learning_rate": 9.307733427958162e-07,
      "loss": 0.0923,
      "num_tokens": 119080670.0,
      "reward": 0.4973958358168602,
      "reward_std": 0.23383906250819564,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 680
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1509.9688110351562,
      "epoch": 0.2615458473355737,
      "grad_norm": 0.255859375,
      "learning_rate": 9.304513078096719e-07,
      "loss": 0.0975,
      "num_tokens": 119238067.0,
      "reward": 0.5781250298023224,
      "reward_std": 0.24977797642350197,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 681
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1487.8541870117188,
      "epoch": 0.2619299087854057,
      "grad_norm": 0.2265625,
      "learning_rate": 9.301285882423091e-07,
      "loss": 0.1117,
      "num_tokens": 119397661.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.17693296261131763,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 682
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1495.0729370117188,
      "epoch": 0.2623139702352376,
      "grad_norm": 0.251953125,
      "learning_rate": 9.298051846744269e-07,
      "loss": 0.1293,
      "num_tokens": 119558492.0,
      "reward": 0.6510417014360428,
      "reward_std": 0.3321352005004883,
      "rewards/accuracy_reward": 0.18750000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416641831398,
      "step": 683
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.2604675292969,
      "epoch": 0.2626980316850696,
      "grad_norm": 0.205078125,
      "learning_rate": 9.294810976879554e-07,
      "loss": 0.074,
      "num_tokens": 119741237.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.2466724757105112,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 684
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.0625305175781,
      "epoch": 0.2630820931349016,
      "grad_norm": 0.1953125,
      "learning_rate": 9.291563278660538e-07,
      "loss": 0.067,
      "num_tokens": 119916043.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.19250460155308247,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 685
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.2709045410156,
      "epoch": 0.26346615458473355,
      "grad_norm": 0.2041015625,
      "learning_rate": 9.288308757931109e-07,
      "loss": 0.0729,
      "num_tokens": 120099421.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.25141144543886185,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 686
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1277.2500457763672,
      "epoch": 0.26385021603456554,
      "grad_norm": 0.25390625,
      "learning_rate": 9.285047420547426e-07,
      "loss": 0.1135,
      "num_tokens": 120237621.0,
      "reward": 0.6302083432674408,
      "reward_std": 0.24503586441278458,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4947916716337204,
      "step": 687
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.8229370117188,
      "epoch": 0.2642342774843975,
      "grad_norm": 0.189453125,
      "learning_rate": 9.281779272377916e-07,
      "loss": 0.0836,
      "num_tokens": 120415140.0,
      "reward": 0.440104179084301,
      "reward_std": 0.13429384864866734,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 688
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.9583740234375,
      "epoch": 0.2646183389342295,
      "grad_norm": 0.2138671875,
      "learning_rate": 9.278504319303258e-07,
      "loss": 0.0711,
      "num_tokens": 120586872.0,
      "reward": 0.447916679084301,
      "reward_std": 0.2403341308236122,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3750000074505806,
      "step": 689
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1426.1042175292969,
      "epoch": 0.2650024003840615,
      "grad_norm": 0.296875,
      "learning_rate": 9.275222567216379e-07,
      "loss": 0.0699,
      "num_tokens": 120736970.0,
      "reward": 0.6692708432674408,
      "reward_std": 0.206652469933033,
      "rewards/accuracy_reward": 0.2499999925494194,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 690
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1845.5000610351562,
      "epoch": 0.2653864618338934,
      "grad_norm": 0.18359375,
      "learning_rate": 9.27193402202244e-07,
      "loss": 0.084,
      "num_tokens": 120927666.0,
      "reward": 0.479166679084301,
      "reward_std": 0.20843207836151123,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.34375,
      "step": 691
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1489.0625610351562,
      "epoch": 0.2657705232837254,
      "grad_norm": 0.21484375,
      "learning_rate": 9.268638689638824e-07,
      "loss": 0.0541,
      "num_tokens": 121086056.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.24614083394408226,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.5416870117188,
      "epoch": 0.26615458473355735,
      "grad_norm": 0.2001953125,
      "learning_rate": 9.265336575995125e-07,
      "loss": 0.1012,
      "num_tokens": 121252692.0,
      "reward": 0.513020858168602,
      "reward_std": 0.2596090957522392,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 693
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.9375305175781,
      "epoch": 0.26653864618338935,
      "grad_norm": 0.189453125,
      "learning_rate": 9.262027687033144e-07,
      "loss": 0.1316,
      "num_tokens": 121435366.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.21198060736060143,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 694
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.3750610351562,
      "epoch": 0.2669227076332213,
      "grad_norm": 0.2314453125,
      "learning_rate": 9.258712028706871e-07,
      "loss": 0.0787,
      "num_tokens": 121603858.0,
      "reward": 0.5651041865348816,
      "reward_std": 0.29408515989780426,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875074505806,
      "step": 695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.0729370117188,
      "epoch": 0.2673067690830533,
      "grad_norm": 0.2255859375,
      "learning_rate": 9.255389606982475e-07,
      "loss": 0.0996,
      "num_tokens": 121774713.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.15293232537806034,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 696
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1487.8541870117188,
      "epoch": 0.2676908305328853,
      "grad_norm": 0.2216796875,
      "learning_rate": 9.252060427838296e-07,
      "loss": 0.0789,
      "num_tokens": 121932387.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.2315693460404873,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 697
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.8645935058594,
      "epoch": 0.2680748919827172,
      "grad_norm": 0.1806640625,
      "learning_rate": 9.248724497264838e-07,
      "loss": 0.0782,
      "num_tokens": 122122950.0,
      "reward": 0.3567708432674408,
      "reward_std": 0.12352717388421297,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375074505806,
      "step": 698
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1726.7604675292969,
      "epoch": 0.2684589534325492,
      "grad_norm": 0.18359375,
      "learning_rate": 9.245381821264745e-07,
      "loss": 0.0893,
      "num_tokens": 122301327.0,
      "reward": 0.4869791939854622,
      "reward_std": 0.2070555416867137,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 699
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.1250305175781,
      "epoch": 0.26884301488238116,
      "grad_norm": 0.232421875,
      "learning_rate": 9.242032405852806e-07,
      "loss": 0.0528,
      "num_tokens": 122481043.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.17178875394165516,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 700
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.0104370117188,
      "epoch": 0.26922707633221316,
      "grad_norm": 0.201171875,
      "learning_rate": 9.238676257055931e-07,
      "loss": 0.07,
      "num_tokens": 122662820.0,
      "reward": 0.6536458432674408,
      "reward_std": 0.2921255975961685,
      "rewards/accuracy_reward": 0.2499999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 701
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1462.3750610351562,
      "epoch": 0.26961113778204515,
      "grad_norm": 0.16015625,
      "learning_rate": 9.235313380913149e-07,
      "loss": 0.096,
      "num_tokens": 122816024.0,
      "reward": 0.463541679084301,
      "reward_std": 0.1603718362748623,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 702
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.5312805175781,
      "epoch": 0.2699951992318771,
      "grad_norm": 0.2294921875,
      "learning_rate": 9.231943783475599e-07,
      "loss": 0.0311,
      "num_tokens": 122989099.0,
      "reward": 0.4557291716337204,
      "reward_std": 0.23062435071915388,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 703
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.8229675292969,
      "epoch": 0.2703792606817091,
      "grad_norm": 0.20703125,
      "learning_rate": 9.228567470806503e-07,
      "loss": 0.094,
      "num_tokens": 123167834.0,
      "reward": 0.3984375074505806,
      "reward_std": 0.173091609030962,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 704
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.8125305175781,
      "epoch": 0.27076332213154103,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.225184448981176e-07,
      "loss": 0.131,
      "num_tokens": 123345408.0,
      "reward": 0.4583333507180214,
      "reward_std": 0.2127695195376873,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 705
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.1146240234375,
      "epoch": 0.271147383581373,
      "grad_norm": 0.2275390625,
      "learning_rate": 9.221794724087001e-07,
      "loss": 0.0686,
      "num_tokens": 123529699.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.1304546222090721,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333432674408,
      "step": 706
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.8646240234375,
      "epoch": 0.27153144503120497,
      "grad_norm": 0.22265625,
      "learning_rate": 9.218398302223423e-07,
      "loss": 0.0664,
      "num_tokens": 123683606.0,
      "reward": 0.5208333358168602,
      "reward_std": 0.17136747390031815,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 707
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.6354370117188,
      "epoch": 0.27191550648103696,
      "grad_norm": 0.1845703125,
      "learning_rate": 9.214995189501941e-07,
      "loss": 0.0782,
      "num_tokens": 123865275.0,
      "reward": 0.4140625149011612,
      "reward_std": 0.1984054483473301,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 708
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.3750305175781,
      "epoch": 0.27229956793086896,
      "grad_norm": 0.212890625,
      "learning_rate": 9.211585392046087e-07,
      "loss": 0.0928,
      "num_tokens": 124045927.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.23563451506197453,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083432674408,
      "step": 709
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.6250305175781,
      "epoch": 0.2726836293807009,
      "grad_norm": 0.2197265625,
      "learning_rate": 9.208168915991425e-07,
      "loss": 0.0947,
      "num_tokens": 124214491.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.23325365781784058,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 710
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.1563110351562,
      "epoch": 0.2730676908305329,
      "grad_norm": 0.220703125,
      "learning_rate": 9.204745767485536e-07,
      "loss": 0.0911,
      "num_tokens": 124378506.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.2217888068407774,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 711
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.0625305175781,
      "epoch": 0.27345175228036483,
      "grad_norm": 0.1826171875,
      "learning_rate": 9.201315952688012e-07,
      "loss": 0.0063,
      "num_tokens": 124546992.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.1372080435976386,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 712
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1440.1666717529297,
      "epoch": 0.27383581373019683,
      "grad_norm": 0.259765625,
      "learning_rate": 9.19787947777043e-07,
      "loss": 0.0953,
      "num_tokens": 124698544.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.15569245629012585,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 713
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1372.7396240234375,
      "epoch": 0.2742198751800288,
      "grad_norm": 0.21484375,
      "learning_rate": 9.194436348916363e-07,
      "loss": 0.0717,
      "num_tokens": 124841951.0,
      "reward": 0.6354166865348816,
      "reward_std": 0.19833670929074287,
      "rewards/accuracy_reward": 0.17708333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 714
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.9375305175781,
      "epoch": 0.27460393662986077,
      "grad_norm": 0.2431640625,
      "learning_rate": 9.190986572321344e-07,
      "loss": 0.131,
      "num_tokens": 125012681.0,
      "reward": 0.463541679084301,
      "reward_std": 0.2784346826374531,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 715
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.5833435058594,
      "epoch": 0.27498799807969276,
      "grad_norm": 0.2216796875,
      "learning_rate": 9.187530154192882e-07,
      "loss": 0.1576,
      "num_tokens": 125183033.0,
      "reward": 0.479166679084301,
      "reward_std": 0.263100553303957,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 716
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.7604370117188,
      "epoch": 0.2753720595295247,
      "grad_norm": 0.1845703125,
      "learning_rate": 9.184067100750425e-07,
      "loss": 0.073,
      "num_tokens": 125366242.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.14195415982976556,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671874925494194,
      "step": 717
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.1354675292969,
      "epoch": 0.2757561209793567,
      "grad_norm": 0.220703125,
      "learning_rate": 9.180597418225369e-07,
      "loss": 0.1087,
      "num_tokens": 125531727.0,
      "reward": 0.4973958358168602,
      "reward_std": 0.2507116496562958,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 718
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1850.1042175292969,
      "epoch": 0.2761401824291887,
      "grad_norm": 0.173828125,
      "learning_rate": 9.177121112861031e-07,
      "loss": 0.105,
      "num_tokens": 125724249.0,
      "reward": 0.3906250074505806,
      "reward_std": 0.16058621369302273,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 719
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.3646240234375,
      "epoch": 0.27652424387902064,
      "grad_norm": 0.197265625,
      "learning_rate": 9.173638190912651e-07,
      "loss": 0.1233,
      "num_tokens": 125897900.0,
      "reward": 0.518229179084301,
      "reward_std": 0.2831853814423084,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 720
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1909.2396240234375,
      "epoch": 0.27690830532885263,
      "grad_norm": 0.1591796875,
      "learning_rate": 9.170148658647372e-07,
      "loss": 0.0931,
      "num_tokens": 126096659.0,
      "reward": 0.3333333432674408,
      "reward_std": 0.1247520474717021,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3125,
      "step": 721
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1290.8541870117188,
      "epoch": 0.2772923667786846,
      "grad_norm": 0.232421875,
      "learning_rate": 9.166652522344231e-07,
      "loss": 0.1117,
      "num_tokens": 126232453.0,
      "reward": 0.640625,
      "reward_std": 0.30376999033614993,
      "rewards/accuracy_reward": 0.1874999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531249925494194,
      "step": 722
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1554.8021240234375,
      "epoch": 0.27767642822851657,
      "grad_norm": 0.263671875,
      "learning_rate": 9.163149788294152e-07,
      "loss": 0.0961,
      "num_tokens": 126398538.0,
      "reward": 0.5130208358168602,
      "reward_std": 0.22062614560127258,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 723
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.6042175292969,
      "epoch": 0.2780604896783485,
      "grad_norm": 0.236328125,
      "learning_rate": 9.159640462799925e-07,
      "loss": 0.1241,
      "num_tokens": 126574564.0,
      "reward": 0.393229179084301,
      "reward_std": 0.12993087619543076,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 724
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.8541870117188,
      "epoch": 0.2784445511281805,
      "grad_norm": 0.212890625,
      "learning_rate": 9.156124552176205e-07,
      "loss": 0.1133,
      "num_tokens": 126745270.0,
      "reward": 0.4583333507180214,
      "reward_std": 0.26124412938952446,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 725
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.1458740234375,
      "epoch": 0.2788286125780125,
      "grad_norm": 0.236328125,
      "learning_rate": 9.152602062749497e-07,
      "loss": 0.0488,
      "num_tokens": 126898796.0,
      "reward": 0.559895858168602,
      "reward_std": 0.297635231167078,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 726
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1349.9896240234375,
      "epoch": 0.27921267402784444,
      "grad_norm": 0.279296875,
      "learning_rate": 9.149073000858141e-07,
      "loss": 0.0797,
      "num_tokens": 127041627.0,
      "reward": 0.5963541939854622,
      "reward_std": 0.2673543319106102,
      "rewards/accuracy_reward": 0.14583333022892475,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 727
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.7396240234375,
      "epoch": 0.27959673547767644,
      "grad_norm": 0.30859375,
      "learning_rate": 9.145537372852302e-07,
      "loss": 0.0413,
      "num_tokens": 127210306.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.21790491044521332,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 728
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1648.7396240234375,
      "epoch": 0.2799807969275084,
      "grad_norm": 0.263671875,
      "learning_rate": 9.141995185093966e-07,
      "loss": 0.1312,
      "num_tokens": 127382801.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.22718001157045364,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 729
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1474.6979217529297,
      "epoch": 0.2803648583773404,
      "grad_norm": 0.236328125,
      "learning_rate": 9.138446443956916e-07,
      "loss": 0.0242,
      "num_tokens": 127536580.0,
      "reward": 0.5442708507180214,
      "reward_std": 0.21348807029426098,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 730
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.8125,
      "epoch": 0.28074891982717237,
      "grad_norm": 0.228515625,
      "learning_rate": 9.134891155826731e-07,
      "loss": 0.1155,
      "num_tokens": 127718130.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.23826528526842594,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 731
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1432.0521240234375,
      "epoch": 0.2811329812770043,
      "grad_norm": 0.267578125,
      "learning_rate": 9.131329327100772e-07,
      "loss": 0.1089,
      "num_tokens": 127869239.0,
      "reward": 0.674479179084301,
      "reward_std": 0.31313716620206833,
      "rewards/accuracy_reward": 0.2083333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 732
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.3229370117188,
      "epoch": 0.2815170427268363,
      "grad_norm": 0.212890625,
      "learning_rate": 9.127760964188164e-07,
      "loss": 0.0895,
      "num_tokens": 128047182.0,
      "reward": 0.3515625074505806,
      "reward_std": 0.0803128220140934,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 733
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1380.8542175292969,
      "epoch": 0.28190110417666825,
      "grad_norm": 0.333984375,
      "learning_rate": 9.124186073509792e-07,
      "loss": 0.0433,
      "num_tokens": 128195368.0,
      "reward": 0.5208333507180214,
      "reward_std": 0.21654811035841703,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 734
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.4688110351562,
      "epoch": 0.28228516562650025,
      "grad_norm": 0.232421875,
      "learning_rate": 9.12060466149829e-07,
      "loss": 0.0412,
      "num_tokens": 128367605.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.19504400342702866,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 735
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.6250305175781,
      "epoch": 0.2826692270763322,
      "grad_norm": 0.2041015625,
      "learning_rate": 9.117016734598022e-07,
      "loss": 0.0696,
      "num_tokens": 128552913.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.22093750536441803,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458432674408,
      "step": 736
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1365.3333587646484,
      "epoch": 0.2830532885261642,
      "grad_norm": 0.2578125,
      "learning_rate": 9.113422299265077e-07,
      "loss": 0.1722,
      "num_tokens": 128698369.0,
      "reward": 0.614583358168602,
      "reward_std": 0.319159921258688,
      "rewards/accuracy_reward": 0.1249999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4895833358168602,
      "step": 737
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.5313110351562,
      "epoch": 0.2834373499759962,
      "grad_norm": 0.19921875,
      "learning_rate": 9.109821361967255e-07,
      "loss": 0.0472,
      "num_tokens": 128888948.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.12847613357007504,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791567325592,
      "step": 738
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.5312805175781,
      "epoch": 0.2838214114258281,
      "grad_norm": 0.1875,
      "learning_rate": 9.106213929184055e-07,
      "loss": 0.0938,
      "num_tokens": 129064487.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.14864650182425976,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 739
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1841.1875610351562,
      "epoch": 0.2842054728756601,
      "grad_norm": 0.1767578125,
      "learning_rate": 9.102600007406665e-07,
      "loss": 0.0591,
      "num_tokens": 129254489.0,
      "reward": 0.3567708432674408,
      "reward_std": 0.11038343887776136,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 740
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.2917175292969,
      "epoch": 0.28458953432549206,
      "grad_norm": 0.205078125,
      "learning_rate": 9.098979603137949e-07,
      "loss": 0.0831,
      "num_tokens": 129436589.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.1811955664306879,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 741
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.4479675292969,
      "epoch": 0.28497359577532405,
      "grad_norm": 0.1875,
      "learning_rate": 9.095352722892434e-07,
      "loss": 0.1034,
      "num_tokens": 129616992.0,
      "reward": 0.5026041716337204,
      "reward_std": 0.21133921295404434,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 742
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1906.5104675292969,
      "epoch": 0.28535765722515605,
      "grad_norm": 0.1689453125,
      "learning_rate": 9.091719373196302e-07,
      "loss": 0.0486,
      "num_tokens": 129814385.0,
      "reward": 0.3333333358168602,
      "reward_std": 0.10942199220880866,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3229166641831398,
      "step": 743
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1789.0729675292969,
      "epoch": 0.285741718674988,
      "grad_norm": 0.193359375,
      "learning_rate": 9.088079560587372e-07,
      "loss": 0.1135,
      "num_tokens": 129999752.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.16802118346095085,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 744
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.4375,
      "epoch": 0.28612578012482,
      "grad_norm": 0.25,
      "learning_rate": 9.084433291615101e-07,
      "loss": 0.0696,
      "num_tokens": 130167130.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.17654473707079887,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 745
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.0208740234375,
      "epoch": 0.2865098415746519,
      "grad_norm": 0.2197265625,
      "learning_rate": 9.080780572840555e-07,
      "loss": 0.0685,
      "num_tokens": 130333908.0,
      "reward": 0.5364583507180214,
      "reward_std": 0.12138949707150459,
      "rewards/accuracy_reward": 0.1249999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 746
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.2708740234375,
      "epoch": 0.2868939030244839,
      "grad_norm": 0.23828125,
      "learning_rate": 9.077121410836407e-07,
      "loss": 0.0999,
      "num_tokens": 130494342.0,
      "reward": 0.5963541865348816,
      "reward_std": 0.24593372270464897,
      "rewards/accuracy_reward": 0.1770833283662796,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 747
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.2708740234375,
      "epoch": 0.2872779644743159,
      "grad_norm": 0.1962890625,
      "learning_rate": 9.073455812186928e-07,
      "loss": 0.097,
      "num_tokens": 130655592.0,
      "reward": 0.4817708432674408,
      "reward_std": 0.1816980643197894,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041716337204,
      "step": 748
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.0625305175781,
      "epoch": 0.28766202592414786,
      "grad_norm": 0.177734375,
      "learning_rate": 9.069783783487967e-07,
      "loss": 0.0765,
      "num_tokens": 130825606.0,
      "reward": 0.4791666716337204,
      "reward_std": 0.19327502511441708,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 749
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.4896240234375,
      "epoch": 0.28804608737397985,
      "grad_norm": 0.234375,
      "learning_rate": 9.066105331346947e-07,
      "loss": 0.1567,
      "num_tokens": 130989581.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.19031278230249882,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 750
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1426.7083740234375,
      "epoch": 0.2884301488238118,
      "grad_norm": 0.3125,
      "learning_rate": 9.062420462382845e-07,
      "loss": 0.0618,
      "num_tokens": 131141561.0,
      "reward": 0.5312500298023224,
      "reward_std": 0.22121290862560272,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 751
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1351.3229675292969,
      "epoch": 0.2888142102736438,
      "grad_norm": 0.251953125,
      "learning_rate": 9.058729183226189e-07,
      "loss": 0.1002,
      "num_tokens": 131287640.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.20909722708165646,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 752
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.1666870117188,
      "epoch": 0.28919827172347573,
      "grad_norm": 0.19921875,
      "learning_rate": 9.055031500519034e-07,
      "loss": 0.121,
      "num_tokens": 131462608.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.175276767462492,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 753
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.9166870117188,
      "epoch": 0.2895823331733077,
      "grad_norm": 0.158203125,
      "learning_rate": 9.051327420914966e-07,
      "loss": 0.0632,
      "num_tokens": 131648376.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.1187042105011642,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.9271545410156,
      "epoch": 0.2899663946231397,
      "grad_norm": 0.2490234375,
      "learning_rate": 9.04761695107908e-07,
      "loss": 0.1663,
      "num_tokens": 131810585.0,
      "reward": 0.510416679084301,
      "reward_std": 0.20684584975242615,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 755
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1494.6250305175781,
      "epoch": 0.29035045607297166,
      "grad_norm": 0.2294921875,
      "learning_rate": 9.043900097687963e-07,
      "loss": 0.0821,
      "num_tokens": 131968629.0,
      "reward": 0.450520858168602,
      "reward_std": 0.16774627938866615,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874850988388,
      "step": 756
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.0000305175781,
      "epoch": 0.29073451752280366,
      "grad_norm": 0.2119140625,
      "learning_rate": 9.040176867429694e-07,
      "loss": 0.1263,
      "num_tokens": 132145589.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.1706204479560256,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 757
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.6250610351562,
      "epoch": 0.2911185789726356,
      "grad_norm": 0.17578125,
      "learning_rate": 9.036447267003828e-07,
      "loss": 0.0451,
      "num_tokens": 132333873.0,
      "reward": 0.393229179084301,
      "reward_std": 0.20143183879554272,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 758
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.3646240234375,
      "epoch": 0.2915026404224676,
      "grad_norm": 0.1962890625,
      "learning_rate": 9.032711303121379e-07,
      "loss": 0.0724,
      "num_tokens": 132500620.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.1626388244330883,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 759
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.1979675292969,
      "epoch": 0.2918867018722996,
      "grad_norm": 0.251953125,
      "learning_rate": 9.028968982504809e-07,
      "loss": 0.0712,
      "num_tokens": 132655559.0,
      "reward": 0.5885416865348816,
      "reward_std": 0.2661716379225254,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416641831398,
      "step": 760
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1228.6250457763672,
      "epoch": 0.29227076332213153,
      "grad_norm": 0.357421875,
      "learning_rate": 9.025220311888025e-07,
      "loss": 0.1137,
      "num_tokens": 132789475.0,
      "reward": 0.6119791865348816,
      "reward_std": 0.26103927940130234,
      "rewards/accuracy_reward": 0.13541666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 761
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.9583740234375,
      "epoch": 0.29265482477196353,
      "grad_norm": 0.19140625,
      "learning_rate": 9.021465298016357e-07,
      "loss": 0.0948,
      "num_tokens": 132963511.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.23764261603355408,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 762
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.2396240234375,
      "epoch": 0.29303888622179547,
      "grad_norm": 0.2255859375,
      "learning_rate": 9.017703947646547e-07,
      "loss": 0.0409,
      "num_tokens": 133147910.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.16128134354948997,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 763
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.6875305175781,
      "epoch": 0.29342294767162747,
      "grad_norm": 0.2001953125,
      "learning_rate": 9.013936267546742e-07,
      "loss": 0.0536,
      "num_tokens": 133327992.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.2256493978202343,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 764
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.7604675292969,
      "epoch": 0.2938070091214594,
      "grad_norm": 0.2236328125,
      "learning_rate": 9.010162264496475e-07,
      "loss": 0.0866,
      "num_tokens": 133498681.0,
      "reward": 0.5078125223517418,
      "reward_std": 0.23922636918723583,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 765
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1514.9792175292969,
      "epoch": 0.2941910705712914,
      "grad_norm": 0.232421875,
      "learning_rate": 9.00638194528666e-07,
      "loss": 0.0833,
      "num_tokens": 133656855.0,
      "reward": 0.4479166716337204,
      "reward_std": 0.1921222098171711,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833432674408,
      "step": 766
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.9166870117188,
      "epoch": 0.2945751320211234,
      "grad_norm": 0.193359375,
      "learning_rate": 9.002595316719575e-07,
      "loss": 0.071,
      "num_tokens": 133836567.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.16909607127308846,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 767
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.1250610351562,
      "epoch": 0.29495919347095534,
      "grad_norm": 0.2578125,
      "learning_rate": 8.998802385608852e-07,
      "loss": 0.0601,
      "num_tokens": 134008067.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.23533313535153866,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 768
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1931.7917175292969,
      "epoch": 0.29534325492078733,
      "grad_norm": 0.1767578125,
      "learning_rate": 8.995003158779461e-07,
      "loss": 0.0695,
      "num_tokens": 134208831.0,
      "reward": 0.3723958507180214,
      "reward_std": 0.15968158468604088,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458283662796,
      "step": 769
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1761.6458740234375,
      "epoch": 0.2957273163706193,
      "grad_norm": 0.1875,
      "learning_rate": 8.9911976430677e-07,
      "loss": 0.0858,
      "num_tokens": 134394197.0,
      "reward": 0.3958333507180214,
      "reward_std": 0.15483424440026283,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 770
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.6458740234375,
      "epoch": 0.29611137782045127,
      "grad_norm": 0.2138671875,
      "learning_rate": 8.987385845321191e-07,
      "loss": 0.1331,
      "num_tokens": 134556963.0,
      "reward": 0.4947916716337204,
      "reward_std": 0.2256440967321396,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 771
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.5000305175781,
      "epoch": 0.29649543927028327,
      "grad_norm": 0.1640625,
      "learning_rate": 8.98356777239885e-07,
      "loss": 0.0713,
      "num_tokens": 134744947.0,
      "reward": 0.3802083432674408,
      "reward_std": 0.12614583177492023,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 772
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.5312805175781,
      "epoch": 0.2968795007201152,
      "grad_norm": 0.30859375,
      "learning_rate": 8.97974343117089e-07,
      "loss": 0.1654,
      "num_tokens": 134915822.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.18815737403929234,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 773
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1524.1562805175781,
      "epoch": 0.2972635621699472,
      "grad_norm": 0.2578125,
      "learning_rate": 8.9759128285188e-07,
      "loss": 0.1047,
      "num_tokens": 135077605.0,
      "reward": 0.5260416716337204,
      "reward_std": 0.23739951103925705,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250074505806,
      "step": 774
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.5833740234375,
      "epoch": 0.29764762361977914,
      "grad_norm": 0.1875,
      "learning_rate": 8.972075971335342e-07,
      "loss": 0.0365,
      "num_tokens": 135268285.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.18027419177815318,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671874925494194,
      "step": 775
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1645.9792175292969,
      "epoch": 0.29803168506961114,
      "grad_norm": 0.201171875,
      "learning_rate": 8.968232866524523e-07,
      "loss": 0.1373,
      "num_tokens": 135439755.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.18396339938044548,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.4271545410156,
      "epoch": 0.29841574651944314,
      "grad_norm": 0.2001953125,
      "learning_rate": 8.9643835210016e-07,
      "loss": 0.1056,
      "num_tokens": 135625444.0,
      "reward": 0.416666679084301,
      "reward_std": 0.18471965566277504,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 777
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.9895935058594,
      "epoch": 0.2987998079692751,
      "grad_norm": 0.2138671875,
      "learning_rate": 8.960527941693055e-07,
      "loss": 0.1078,
      "num_tokens": 135794347.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.251472819596529,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 778
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.8750610351562,
      "epoch": 0.2991838694191071,
      "grad_norm": 0.19140625,
      "learning_rate": 8.956666135536588e-07,
      "loss": 0.0713,
      "num_tokens": 135974959.0,
      "reward": 0.393229179084301,
      "reward_std": 0.13184381928294897,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 779
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1829.8021240234375,
      "epoch": 0.299567930868939,
      "grad_norm": 0.2001953125,
      "learning_rate": 8.952798109481105e-07,
      "loss": 0.0952,
      "num_tokens": 136165948.0,
      "reward": 0.3984375149011612,
      "reward_std": 0.18398152478039265,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 780
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.9167175292969,
      "epoch": 0.299951992318771,
      "grad_norm": 0.1904296875,
      "learning_rate": 8.948923870486703e-07,
      "loss": 0.0862,
      "num_tokens": 136338764.0,
      "reward": 0.432291679084301,
      "reward_std": 0.15083661582320929,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 781
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.5000305175781,
      "epoch": 0.30033605376860295,
      "grad_norm": 0.361328125,
      "learning_rate": 8.945043425524657e-07,
      "loss": 0.049,
      "num_tokens": 136506716.0,
      "reward": 0.5442708507180214,
      "reward_std": 0.26058050990104675,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 782
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.8854370117188,
      "epoch": 0.30072011521843495,
      "grad_norm": 0.16015625,
      "learning_rate": 8.941156781577412e-07,
      "loss": 0.0614,
      "num_tokens": 136682065.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.09519422007724643,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 783
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1313.5,
      "epoch": 0.30110417666826694,
      "grad_norm": 0.2578125,
      "learning_rate": 8.937263945638565e-07,
      "loss": 0.0837,
      "num_tokens": 136822737.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.27879390493035316,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166567325592,
      "step": 784
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.1875305175781,
      "epoch": 0.3014882381180989,
      "grad_norm": 0.18359375,
      "learning_rate": 8.933364924712858e-07,
      "loss": 0.0887,
      "num_tokens": 136992275.0,
      "reward": 0.4296875,
      "reward_std": 0.15256858617067337,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 785
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.0938110351562,
      "epoch": 0.3018722995679309,
      "grad_norm": 0.2158203125,
      "learning_rate": 8.929459725816156e-07,
      "loss": 0.0978,
      "num_tokens": 137165260.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.18924973905086517,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.6354675292969,
      "epoch": 0.3022563610177628,
      "grad_norm": 0.2119140625,
      "learning_rate": 8.925548355975446e-07,
      "loss": 0.0955,
      "num_tokens": 137335953.0,
      "reward": 0.4505208358168602,
      "reward_std": 0.16792803816497326,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 787
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.1042175292969,
      "epoch": 0.3026404224675948,
      "grad_norm": 0.1728515625,
      "learning_rate": 8.921630822228818e-07,
      "loss": 0.0514,
      "num_tokens": 137514675.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.17859404534101486,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 788
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.9791870117188,
      "epoch": 0.3030244839174268,
      "grad_norm": 0.2216796875,
      "learning_rate": 8.917707131625453e-07,
      "loss": 0.0661,
      "num_tokens": 137680929.0,
      "reward": 0.5468750149011612,
      "reward_std": 0.2573950793594122,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 789
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1656.2187805175781,
      "epoch": 0.30340854536725875,
      "grad_norm": 0.2138671875,
      "learning_rate": 8.913777291225612e-07,
      "loss": 0.1203,
      "num_tokens": 137852558.0,
      "reward": 0.5598958507180214,
      "reward_std": 0.32093676179647446,
      "rewards/accuracy_reward": 0.15624999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 790
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.7292175292969,
      "epoch": 0.30379260681709075,
      "grad_norm": 0.212890625,
      "learning_rate": 8.909841308100614e-07,
      "loss": 0.0562,
      "num_tokens": 138031636.0,
      "reward": 0.4322916865348816,
      "reward_std": 0.17995265498757362,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 791
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.2812805175781,
      "epoch": 0.3041766682669227,
      "grad_norm": 0.302734375,
      "learning_rate": 8.905899189332844e-07,
      "loss": 0.0245,
      "num_tokens": 138218599.0,
      "reward": 0.4531250074505806,
      "reward_std": 0.18170947581529617,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 792
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1398.0937805175781,
      "epoch": 0.3045607297167547,
      "grad_norm": 0.310546875,
      "learning_rate": 8.901950942015718e-07,
      "loss": 0.1025,
      "num_tokens": 138369152.0,
      "reward": 0.5546875223517418,
      "reward_std": 0.2618713341653347,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 793
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.8750305175781,
      "epoch": 0.3049447911665866,
      "grad_norm": 0.2373046875,
      "learning_rate": 8.897996573253682e-07,
      "loss": 0.0699,
      "num_tokens": 138536788.0,
      "reward": 0.4583333507180214,
      "reward_std": 0.17747536674141884,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 794
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1742.9375305175781,
      "epoch": 0.3053288526164186,
      "grad_norm": 0.197265625,
      "learning_rate": 8.894036090162199e-07,
      "loss": 0.11,
      "num_tokens": 138718678.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.18958423659205437,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 795
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.3229675292969,
      "epoch": 0.3057129140662506,
      "grad_norm": 0.1884765625,
      "learning_rate": 8.89006949986773e-07,
      "loss": 0.0572,
      "num_tokens": 138904037.0,
      "reward": 0.5338541939854622,
      "reward_std": 0.2940753884613514,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 796
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.3958740234375,
      "epoch": 0.30609697551608256,
      "grad_norm": 0.1962890625,
      "learning_rate": 8.886096809507731e-07,
      "loss": 0.0695,
      "num_tokens": 139080115.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.183658417314291,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 797
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1832.9062805175781,
      "epoch": 0.30648103696591456,
      "grad_norm": 0.19921875,
      "learning_rate": 8.882118026230631e-07,
      "loss": 0.0731,
      "num_tokens": 139271362.0,
      "reward": 0.3802083432674408,
      "reward_std": 0.1374889276921749,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583283662796,
      "step": 798
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1289.0729675292969,
      "epoch": 0.3068650984157465,
      "grad_norm": 0.255859375,
      "learning_rate": 8.878133157195817e-07,
      "loss": 0.1128,
      "num_tokens": 139409729.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.15552260167896748,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125074505806,
      "step": 799
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.1979675292969,
      "epoch": 0.3072491598655785,
      "grad_norm": 0.2421875,
      "learning_rate": 8.87414220957364e-07,
      "loss": 0.0747,
      "num_tokens": 139572932.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.1685920897871256,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453124925494194,
      "step": 800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.5833740234375,
      "epoch": 0.3076332213154105,
      "grad_norm": 0.2353515625,
      "learning_rate": 8.870145190545376e-07,
      "loss": 0.1163,
      "num_tokens": 139737428.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.22213084995746613,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 801
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.6667175292969,
      "epoch": 0.30801728276524243,
      "grad_norm": 0.2314453125,
      "learning_rate": 8.866142107303233e-07,
      "loss": 0.097,
      "num_tokens": 139905148.0,
      "reward": 0.502604179084301,
      "reward_std": 0.22592981718480587,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 802
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.4167175292969,
      "epoch": 0.3084013442150744,
      "grad_norm": 0.185546875,
      "learning_rate": 8.862132967050329e-07,
      "loss": 0.0507,
      "num_tokens": 140085356.0,
      "reward": 0.5338541865348816,
      "reward_std": 0.27998247742652893,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 803
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.1250305175781,
      "epoch": 0.30878540566490636,
      "grad_norm": 0.2255859375,
      "learning_rate": 8.858117777000679e-07,
      "loss": 0.1147,
      "num_tokens": 140247912.0,
      "reward": 0.513020858168602,
      "reward_std": 0.23476513102650642,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609374925494194,
      "step": 804
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.6354370117188,
      "epoch": 0.30916946711473836,
      "grad_norm": 0.1875,
      "learning_rate": 8.854096544379187e-07,
      "loss": 0.0764,
      "num_tokens": 140424557.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.18628833815455437,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208432674408,
      "step": 805
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.1041870117188,
      "epoch": 0.30955352856457036,
      "grad_norm": 0.1884765625,
      "learning_rate": 8.850069276421628e-07,
      "loss": 0.1186,
      "num_tokens": 140606887.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.2049795649945736,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 806
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.6875610351562,
      "epoch": 0.3099375900144023,
      "grad_norm": 0.1669921875,
      "learning_rate": 8.846035980374638e-07,
      "loss": 0.0557,
      "num_tokens": 140783793.0,
      "reward": 0.5312500223517418,
      "reward_std": 0.19574952125549316,
      "rewards/accuracy_reward": 0.1562500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 807
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.7188110351562,
      "epoch": 0.3103216514642343,
      "grad_norm": 0.2177734375,
      "learning_rate": 8.841996663495702e-07,
      "loss": 0.1451,
      "num_tokens": 140952406.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.25740259885787964,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 808
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.3125305175781,
      "epoch": 0.31070571291406623,
      "grad_norm": 0.162109375,
      "learning_rate": 8.837951333053133e-07,
      "loss": 0.0842,
      "num_tokens": 141140652.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.2519013583660126,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 809
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1858.8854370117188,
      "epoch": 0.31108977436389823,
      "grad_norm": 0.189453125,
      "learning_rate": 8.833899996326069e-07,
      "loss": 0.1068,
      "num_tokens": 141335569.0,
      "reward": 0.455729179084301,
      "reward_std": 0.25008998811244965,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 810
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.1770935058594,
      "epoch": 0.31147383581373017,
      "grad_norm": 0.173828125,
      "learning_rate": 8.829842660604454e-07,
      "loss": 0.1087,
      "num_tokens": 141516042.0,
      "reward": 0.4140625149011612,
      "reward_std": 0.16359908133745193,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291567325592,
      "step": 811
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.6875305175781,
      "epoch": 0.31185789726356217,
      "grad_norm": 0.2041015625,
      "learning_rate": 8.825779333189031e-07,
      "loss": 0.1221,
      "num_tokens": 141692492.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.20948638394474983,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 812
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.3437805175781,
      "epoch": 0.31224195871339416,
      "grad_norm": 0.2001953125,
      "learning_rate": 8.821710021391317e-07,
      "loss": 0.0606,
      "num_tokens": 141859629.0,
      "reward": 0.447916679084301,
      "reward_std": 0.18504899367690086,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 813
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.3645935058594,
      "epoch": 0.3126260201632261,
      "grad_norm": 0.197265625,
      "learning_rate": 8.817634732533601e-07,
      "loss": 0.0836,
      "num_tokens": 142030256.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.267764737829566,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958432674408,
      "step": 814
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.2708740234375,
      "epoch": 0.3130100816130581,
      "grad_norm": 0.2099609375,
      "learning_rate": 8.81355347394893e-07,
      "loss": 0.0789,
      "num_tokens": 142208130.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.1562635637819767,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906250074505806,
      "step": 815
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.7396240234375,
      "epoch": 0.31339414306289004,
      "grad_norm": 0.21484375,
      "learning_rate": 8.809466252981087e-07,
      "loss": 0.1193,
      "num_tokens": 142371025.0,
      "reward": 0.528645858168602,
      "reward_std": 0.23185522388666868,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 816
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.3646240234375,
      "epoch": 0.31377820451272204,
      "grad_norm": 0.1884765625,
      "learning_rate": 8.805373076984588e-07,
      "loss": 0.1091,
      "num_tokens": 142546684.0,
      "reward": 0.408854179084301,
      "reward_std": 0.12950986810028553,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 817
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.5104370117188,
      "epoch": 0.31416226596255403,
      "grad_norm": 0.2001953125,
      "learning_rate": 8.801273953324664e-07,
      "loss": 0.1098,
      "num_tokens": 142725717.0,
      "reward": 0.4479166716337204,
      "reward_std": 0.21228336542844772,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 818
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.5000610351562,
      "epoch": 0.314546327412386,
      "grad_norm": 0.177734375,
      "learning_rate": 8.797168889377245e-07,
      "loss": 0.0637,
      "num_tokens": 142905621.0,
      "reward": 0.4166666716337204,
      "reward_std": 0.14790832810103893,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 819
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.0625305175781,
      "epoch": 0.31493038886221797,
      "grad_norm": 0.1982421875,
      "learning_rate": 8.793057892528955e-07,
      "loss": 0.1295,
      "num_tokens": 143058691.0,
      "reward": 0.5755208432674408,
      "reward_std": 0.24844776466488838,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 820
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1869.0521545410156,
      "epoch": 0.3153144503120499,
      "grad_norm": 0.1845703125,
      "learning_rate": 8.788940970177085e-07,
      "loss": 0.0953,
      "num_tokens": 143250936.0,
      "reward": 0.3385416716337204,
      "reward_std": 0.11110625509172678,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.328125,
      "step": 821
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.4895935058594,
      "epoch": 0.3156985117618819,
      "grad_norm": 0.2109375,
      "learning_rate": 8.784818129729603e-07,
      "loss": 0.1203,
      "num_tokens": 143414239.0,
      "reward": 0.4401041716337204,
      "reward_std": 0.1481885351240635,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 822
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.0937805175781,
      "epoch": 0.3160825732117139,
      "grad_norm": 0.20703125,
      "learning_rate": 8.780689378605109e-07,
      "loss": 0.0996,
      "num_tokens": 143589808.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.1656978353857994,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 823
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.8333740234375,
      "epoch": 0.31646663466154584,
      "grad_norm": 0.2236328125,
      "learning_rate": 8.776554724232853e-07,
      "loss": 0.067,
      "num_tokens": 143763952.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.1934405229985714,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 824
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.2708740234375,
      "epoch": 0.31685069611137784,
      "grad_norm": 0.2197265625,
      "learning_rate": 8.772414174052696e-07,
      "loss": -0.0144,
      "num_tokens": 143943234.0,
      "reward": 0.4114583507180214,
      "reward_std": 0.14892577566206455,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 825
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.1042175292969,
      "epoch": 0.3172347575612098,
      "grad_norm": 0.2197265625,
      "learning_rate": 8.768267735515115e-07,
      "loss": 0.0838,
      "num_tokens": 144112076.0,
      "reward": 0.5312500074505806,
      "reward_std": 0.19064238853752613,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 826
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.1146240234375,
      "epoch": 0.3176188190110418,
      "grad_norm": 0.2138671875,
      "learning_rate": 8.764115416081179e-07,
      "loss": 0.0964,
      "num_tokens": 144292415.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.21179355308413506,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 827
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.4687805175781,
      "epoch": 0.3180028804608737,
      "grad_norm": 0.2099609375,
      "learning_rate": 8.759957223222543e-07,
      "loss": 0.1084,
      "num_tokens": 144474068.0,
      "reward": 0.473958358168602,
      "reward_std": 0.20576537773013115,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 828
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.1771240234375,
      "epoch": 0.3183869419107057,
      "grad_norm": 0.2236328125,
      "learning_rate": 8.755793164421425e-07,
      "loss": 0.0829,
      "num_tokens": 144643597.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.20615305192768574,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 829
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1350.9271240234375,
      "epoch": 0.3187710033605377,
      "grad_norm": 0.353515625,
      "learning_rate": 8.751623247170603e-07,
      "loss": 0.0538,
      "num_tokens": 144785150.0,
      "reward": 0.5364583507180214,
      "reward_std": 0.2169545218348503,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5052083358168602,
      "step": 830
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1498.7187805175781,
      "epoch": 0.31915506481036965,
      "grad_norm": 0.216796875,
      "learning_rate": 8.747447478973395e-07,
      "loss": 0.0791,
      "num_tokens": 144942315.0,
      "reward": 0.5286458432674408,
      "reward_std": 0.18965812027454376,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453124925494194,
      "step": 831
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.0937805175781,
      "epoch": 0.31953912626020164,
      "grad_norm": 0.267578125,
      "learning_rate": 8.743265867343644e-07,
      "loss": 0.0706,
      "num_tokens": 145110292.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.23704486154019833,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 832
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.9688110351562,
      "epoch": 0.3199231877100336,
      "grad_norm": 0.177734375,
      "learning_rate": 8.73907841980571e-07,
      "loss": 0.0974,
      "num_tokens": 145281249.0,
      "reward": 0.455729179084301,
      "reward_std": 0.17910329066216946,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 833
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.4479370117188,
      "epoch": 0.3203072491598656,
      "grad_norm": 0.1884765625,
      "learning_rate": 8.734885143894462e-07,
      "loss": 0.0771,
      "num_tokens": 145458612.0,
      "reward": 0.4843750223517418,
      "reward_std": 0.22035950236022472,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 834
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.0729675292969,
      "epoch": 0.3206913106096976,
      "grad_norm": 0.2158203125,
      "learning_rate": 8.73068604715524e-07,
      "loss": 0.114,
      "num_tokens": 145614331.0,
      "reward": 0.5703125149011612,
      "reward_std": 0.23364506289362907,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.434895820915699,
      "step": 835
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1415.9271240234375,
      "epoch": 0.3210753720595295,
      "grad_norm": 0.23828125,
      "learning_rate": 8.726481137143873e-07,
      "loss": 0.1028,
      "num_tokens": 145763852.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.13613556884229183,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 836
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1667.0833740234375,
      "epoch": 0.3214594335093615,
      "grad_norm": 0.189453125,
      "learning_rate": 8.722270421426642e-07,
      "loss": 0.083,
      "num_tokens": 145935452.0,
      "reward": 0.4843750074505806,
      "reward_std": 0.24939700216054916,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 837
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1683.6042175292969,
      "epoch": 0.32184349495919345,
      "grad_norm": 0.1884765625,
      "learning_rate": 8.718053907580278e-07,
      "loss": 0.0972,
      "num_tokens": 146112126.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.1582462377846241,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 838
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.5625305175781,
      "epoch": 0.32222755640902545,
      "grad_norm": 0.2255859375,
      "learning_rate": 8.713831603191945e-07,
      "loss": 0.0974,
      "num_tokens": 146287644.0,
      "reward": 0.5104166939854622,
      "reward_std": 0.23394258972257376,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 839
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.2396240234375,
      "epoch": 0.3226116178588574,
      "grad_norm": 0.2021484375,
      "learning_rate": 8.709603515859221e-07,
      "loss": 0.0788,
      "num_tokens": 146471307.0,
      "reward": 0.4895833507180214,
      "reward_std": 0.23282707296311855,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 840
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1396.4687805175781,
      "epoch": 0.3229956793086894,
      "grad_norm": 0.271484375,
      "learning_rate": 8.705369653190101e-07,
      "loss": 0.0687,
      "num_tokens": 146622112.0,
      "reward": 0.611979179084301,
      "reward_std": 0.22780434973537922,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125074505806,
      "step": 841
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1357.0521240234375,
      "epoch": 0.3233797407585214,
      "grad_norm": 0.21484375,
      "learning_rate": 8.701130022802962e-07,
      "loss": 0.1073,
      "num_tokens": 146767669.0,
      "reward": 0.518229179084301,
      "reward_std": 0.25213082134723663,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453124925494194,
      "step": 842
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.5417175292969,
      "epoch": 0.3237638022083533,
      "grad_norm": 0.203125,
      "learning_rate": 8.696884632326563e-07,
      "loss": 0.0505,
      "num_tokens": 146932041.0,
      "reward": 0.544270858168602,
      "reward_std": 0.2091290857642889,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 843
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.4687805175781,
      "epoch": 0.3241478636581853,
      "grad_norm": 0.2119140625,
      "learning_rate": 8.692633489400029e-07,
      "loss": 0.1441,
      "num_tokens": 147104422.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.19715206697583199,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 844
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.3646240234375,
      "epoch": 0.32453192510801726,
      "grad_norm": 0.1884765625,
      "learning_rate": 8.688376601672835e-07,
      "loss": 0.0587,
      "num_tokens": 147275401.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.14457771554589272,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.408854179084301,
      "step": 845
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.9583740234375,
      "epoch": 0.32491598655784926,
      "grad_norm": 0.2216796875,
      "learning_rate": 8.68411397680479e-07,
      "loss": 0.0961,
      "num_tokens": 147435309.0,
      "reward": 0.583333358168602,
      "reward_std": 0.24938213638961315,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 846
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.572998046875,
      "epoch": 0.32530004800768125,
      "grad_norm": 0.1728515625,
      "learning_rate": 8.679845622466031e-07,
      "loss": 0.1113,
      "num_tokens": 147615428.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.20665503852069378,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 847
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1873.3021545410156,
      "epoch": 0.3256841094575132,
      "grad_norm": 0.189453125,
      "learning_rate": 8.675571546337003e-07,
      "loss": 0.0926,
      "num_tokens": 147810129.0,
      "reward": 0.3645833432674408,
      "reward_std": 0.1686724405735731,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3437500074505806,
      "step": 848
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.3125305175781,
      "epoch": 0.3260681709073452,
      "grad_norm": 0.23046875,
      "learning_rate": 8.671291756108446e-07,
      "loss": 0.0589,
      "num_tokens": 147965911.0,
      "reward": 0.6979167014360428,
      "reward_std": 0.26142190024256706,
      "rewards/accuracy_reward": 0.27083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 849
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.1354370117188,
      "epoch": 0.32645223235717713,
      "grad_norm": 0.251953125,
      "learning_rate": 8.667006259481381e-07,
      "loss": 0.1354,
      "num_tokens": 148135348.0,
      "reward": 0.4895833432674408,
      "reward_std": 0.22088626772165298,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 850
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.5000610351562,
      "epoch": 0.3268362938070091,
      "grad_norm": 0.2060546875,
      "learning_rate": 8.6627150641671e-07,
      "loss": 0.0548,
      "num_tokens": 148309668.0,
      "reward": 0.4739583358168602,
      "reward_std": 0.19209119956940413,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 851
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.3750305175781,
      "epoch": 0.3272203552568411,
      "grad_norm": 0.2099609375,
      "learning_rate": 8.658418177887147e-07,
      "loss": 0.1053,
      "num_tokens": 148492192.0,
      "reward": 0.3828125074505806,
      "reward_std": 0.13660390675067902,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 852
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1489.1667175292969,
      "epoch": 0.32760441670667306,
      "grad_norm": 0.2294921875,
      "learning_rate": 8.654115608373307e-07,
      "loss": 0.1038,
      "num_tokens": 148649088.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.17291029822081327,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 853
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.7604370117188,
      "epoch": 0.32798847815650506,
      "grad_norm": 0.2001953125,
      "learning_rate": 8.649807363367592e-07,
      "loss": 0.1015,
      "num_tokens": 148834193.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.1546575496904552,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 854
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.3333435058594,
      "epoch": 0.328372539606337,
      "grad_norm": 0.17578125,
      "learning_rate": 8.645493450622223e-07,
      "loss": 0.1063,
      "num_tokens": 149010697.0,
      "reward": 0.416666679084301,
      "reward_std": 0.1574767492711544,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333432674408,
      "step": 855
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1417.8021240234375,
      "epoch": 0.328756601056169,
      "grad_norm": 0.265625,
      "learning_rate": 8.641173877899625e-07,
      "loss": 0.1123,
      "num_tokens": 149159942.0,
      "reward": 0.6562500298023224,
      "reward_std": 0.25972868502140045,
      "rewards/accuracy_reward": 0.2083333283662796,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166567325592,
      "step": 856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.0521240234375,
      "epoch": 0.32914066250600094,
      "grad_norm": 0.2236328125,
      "learning_rate": 8.636848652972403e-07,
      "loss": 0.0932,
      "num_tokens": 149324563.0,
      "reward": 0.5338541939854622,
      "reward_std": 0.18522785790264606,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 857
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1485.5729675292969,
      "epoch": 0.32952472395583293,
      "grad_norm": 0.2216796875,
      "learning_rate": 8.632517783623333e-07,
      "loss": 0.1151,
      "num_tokens": 149479922.0,
      "reward": 0.6119791939854622,
      "reward_std": 0.246359221637249,
      "rewards/accuracy_reward": 0.15625000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 858
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.8854370117188,
      "epoch": 0.32990878540566493,
      "grad_norm": 0.2041015625,
      "learning_rate": 8.628181277645354e-07,
      "loss": 0.0843,
      "num_tokens": 149651943.0,
      "reward": 0.5312500223517418,
      "reward_std": 0.24166908860206604,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 859
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.3229370117188,
      "epoch": 0.33029284685549687,
      "grad_norm": 0.185546875,
      "learning_rate": 8.623839142841535e-07,
      "loss": 0.0725,
      "num_tokens": 149820478.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.25895147398114204,
      "rewards/accuracy_reward": 0.13541666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 860
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.6563110351562,
      "epoch": 0.33067690830532886,
      "grad_norm": 0.173828125,
      "learning_rate": 8.619491387025085e-07,
      "loss": 0.0804,
      "num_tokens": 150011461.0,
      "reward": 0.4192708507180214,
      "reward_std": 0.19440478831529617,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 861
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1463.4792175292969,
      "epoch": 0.3310609697551608,
      "grad_norm": 0.240234375,
      "learning_rate": 8.615138018019323e-07,
      "loss": 0.1062,
      "num_tokens": 150168419.0,
      "reward": 0.5833333507180214,
      "reward_std": 0.28542812541127205,
      "rewards/accuracy_reward": 0.13541667070239782,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 862
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.6458740234375,
      "epoch": 0.3314450312049928,
      "grad_norm": 0.17578125,
      "learning_rate": 8.610779043657665e-07,
      "loss": 0.0601,
      "num_tokens": 150337033.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.11570430686697364,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 863
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1492.0521240234375,
      "epoch": 0.3318290926548248,
      "grad_norm": 0.318359375,
      "learning_rate": 8.606414471783621e-07,
      "loss": 0.102,
      "num_tokens": 150493630.0,
      "reward": 0.5833333507180214,
      "reward_std": 0.2037593126296997,
      "rewards/accuracy_reward": 0.15625000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 864
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.4583740234375,
      "epoch": 0.33221315410465674,
      "grad_norm": 0.2138671875,
      "learning_rate": 8.602044310250767e-07,
      "loss": 0.1086,
      "num_tokens": 150666354.0,
      "reward": 0.4114583507180214,
      "reward_std": 0.17851846292614937,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 865
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.6875610351562,
      "epoch": 0.33259721555448873,
      "grad_norm": 0.212890625,
      "learning_rate": 8.59766856692274e-07,
      "loss": 0.1,
      "num_tokens": 150843748.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.1945810206234455,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 866
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.479248046875,
      "epoch": 0.3329812770043207,
      "grad_norm": 0.2158203125,
      "learning_rate": 8.593287249673218e-07,
      "loss": 0.0746,
      "num_tokens": 151007762.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.11704926332458854,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 867
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1662.2708435058594,
      "epoch": 0.33336533845415267,
      "grad_norm": 0.228515625,
      "learning_rate": 8.588900366385911e-07,
      "loss": 0.1528,
      "num_tokens": 151181068.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.20250878483057022,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 868
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.1145935058594,
      "epoch": 0.3337493999039846,
      "grad_norm": 0.1845703125,
      "learning_rate": 8.584507924954542e-07,
      "loss": 0.095,
      "num_tokens": 151359039.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.17380638420581818,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 869
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.0000305175781,
      "epoch": 0.3341334613538166,
      "grad_norm": 0.345703125,
      "learning_rate": 8.580109933282839e-07,
      "loss": 0.1216,
      "num_tokens": 151526607.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.23632119968533516,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 870
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.7604675292969,
      "epoch": 0.3345175228036486,
      "grad_norm": 0.142578125,
      "learning_rate": 8.575706399284514e-07,
      "loss": 0.0789,
      "num_tokens": 151710728.0,
      "reward": 0.424479179084301,
      "reward_std": 0.23221264779567719,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 871
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.4791870117188,
      "epoch": 0.33490158425348054,
      "grad_norm": 0.1953125,
      "learning_rate": 8.571297330883254e-07,
      "loss": 0.1069,
      "num_tokens": 151886662.0,
      "reward": 0.4114583507180214,
      "reward_std": 0.16191048361361027,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.5938110351562,
      "epoch": 0.33528564570331254,
      "grad_norm": 0.263671875,
      "learning_rate": 8.566882736012703e-07,
      "loss": 0.0937,
      "num_tokens": 152049191.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.26364049687981606,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 873
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.6042175292969,
      "epoch": 0.3356697071531445,
      "grad_norm": 0.203125,
      "learning_rate": 8.562462622616449e-07,
      "loss": 0.0755,
      "num_tokens": 152220073.0,
      "reward": 0.5494791716337204,
      "reward_std": 0.2505226731300354,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 874
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.6979675292969,
      "epoch": 0.3360537686029765,
      "grad_norm": 0.205078125,
      "learning_rate": 8.558036998648013e-07,
      "loss": 0.1049,
      "num_tokens": 152404244.0,
      "reward": 0.458333358168602,
      "reward_std": 0.22965823858976364,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 875
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.7083740234375,
      "epoch": 0.3364378300528085,
      "grad_norm": 0.166015625,
      "learning_rate": 8.553605872070826e-07,
      "loss": 0.0303,
      "num_tokens": 152589240.0,
      "reward": 0.463541679084301,
      "reward_std": 0.1922624846920371,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.3854675292969,
      "epoch": 0.3368218915026404,
      "grad_norm": 0.2490234375,
      "learning_rate": 8.549169250858227e-07,
      "loss": 0.1496,
      "num_tokens": 152760021.0,
      "reward": 0.4348958358168602,
      "reward_std": 0.15698087960481644,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 877
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1869.3541870117188,
      "epoch": 0.3372059529524724,
      "grad_norm": 0.1630859375,
      "learning_rate": 8.544727142993436e-07,
      "loss": 0.0335,
      "num_tokens": 152954727.0,
      "reward": 0.440104179084301,
      "reward_std": 0.13409593934193254,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3151041641831398,
      "step": 878
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.6042175292969,
      "epoch": 0.33759001440230435,
      "grad_norm": 0.2197265625,
      "learning_rate": 8.540279556469552e-07,
      "loss": 0.1251,
      "num_tokens": 153124785.0,
      "reward": 0.533854179084301,
      "reward_std": 0.2970299832522869,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 879
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1415.5104370117188,
      "epoch": 0.33797407585213635,
      "grad_norm": 0.2197265625,
      "learning_rate": 8.535826499289526e-07,
      "loss": 0.104,
      "num_tokens": 153271826.0,
      "reward": 0.5625000223517418,
      "reward_std": 0.17503680475056171,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 880
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1837.0313110351562,
      "epoch": 0.33835813730196834,
      "grad_norm": 0.2001953125,
      "learning_rate": 8.531367979466155e-07,
      "loss": 0.1045,
      "num_tokens": 153464565.0,
      "reward": 0.3645833432674408,
      "reward_std": 0.15412827767431736,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3437499925494194,
      "step": 881
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.0104370117188,
      "epoch": 0.3387421987518003,
      "grad_norm": 0.208984375,
      "learning_rate": 8.52690400502207e-07,
      "loss": 0.0929,
      "num_tokens": 153632982.0,
      "reward": 0.6328125,
      "reward_std": 0.2764206603169441,
      "rewards/accuracy_reward": 0.2291666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 882
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1867.7708435058594,
      "epoch": 0.3391262602016323,
      "grad_norm": 0.166015625,
      "learning_rate": 8.522434583989708e-07,
      "loss": 0.0625,
      "num_tokens": 153827424.0,
      "reward": 0.447916679084301,
      "reward_std": 0.22183488123118877,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 883
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1477.2708740234375,
      "epoch": 0.3395103216514642,
      "grad_norm": 0.240234375,
      "learning_rate": 8.517959724411313e-07,
      "loss": 0.0989,
      "num_tokens": 153985170.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.1451800838112831,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140624925494194,
      "step": 884
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.0729370117188,
      "epoch": 0.3398943831012962,
      "grad_norm": 0.212890625,
      "learning_rate": 8.513479434338915e-07,
      "loss": 0.0832,
      "num_tokens": 154164121.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.16890627518296242,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 885
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.7708740234375,
      "epoch": 0.34027844455112816,
      "grad_norm": 0.21484375,
      "learning_rate": 8.508993721834315e-07,
      "loss": 0.1116,
      "num_tokens": 154344803.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.2199318055063486,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 886
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.7812805175781,
      "epoch": 0.34066250600096015,
      "grad_norm": 0.224609375,
      "learning_rate": 8.504502594969068e-07,
      "loss": 0.1364,
      "num_tokens": 154519286.0,
      "reward": 0.6744792014360428,
      "reward_std": 0.36849891394376755,
      "rewards/accuracy_reward": 0.2708333348855376,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458432674408,
      "step": 887
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.1771545410156,
      "epoch": 0.34104656745079215,
      "grad_norm": 0.2099609375,
      "learning_rate": 8.500006061824475e-07,
      "loss": 0.0881,
      "num_tokens": 154687063.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.21793978288769722,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 888
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.1458740234375,
      "epoch": 0.3414306289006241,
      "grad_norm": 0.19921875,
      "learning_rate": 8.495504130491564e-07,
      "loss": 0.1058,
      "num_tokens": 154869021.0,
      "reward": 0.4635416865348816,
      "reward_std": 0.15871743857860565,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 889
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1723.0208740234375,
      "epoch": 0.3418146903504561,
      "grad_norm": 0.208984375,
      "learning_rate": 8.490996809071075e-07,
      "loss": 0.074,
      "num_tokens": 155049087.0,
      "reward": 0.377604179084301,
      "reward_std": 0.1110756816342473,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 890
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.8646545410156,
      "epoch": 0.342198751800288,
      "grad_norm": 0.2294921875,
      "learning_rate": 8.48648410567345e-07,
      "loss": 0.0811,
      "num_tokens": 155238906.0,
      "reward": 0.3880208358168602,
      "reward_std": 0.21150685846805573,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 891
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.8542175292969,
      "epoch": 0.34258281325012,
      "grad_norm": 0.21484375,
      "learning_rate": 8.481966028418813e-07,
      "loss": 0.1137,
      "num_tokens": 155408916.0,
      "reward": 0.6250000074505806,
      "reward_std": 0.26355355605483055,
      "rewards/accuracy_reward": 0.18750000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 892
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.6042175292969,
      "epoch": 0.342966874699952,
      "grad_norm": 0.2060546875,
      "learning_rate": 8.477442585436958e-07,
      "loss": 0.1274,
      "num_tokens": 155585014.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.24272068589925766,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 893
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.2812805175781,
      "epoch": 0.34335093614978396,
      "grad_norm": 0.1923828125,
      "learning_rate": 8.472913784867333e-07,
      "loss": 0.0811,
      "num_tokens": 155769065.0,
      "reward": 0.416666679084301,
      "reward_std": 0.21093427715823054,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 894
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1509.5208740234375,
      "epoch": 0.34373499759961595,
      "grad_norm": 0.25390625,
      "learning_rate": 8.468379634859028e-07,
      "loss": 0.1242,
      "num_tokens": 155927947.0,
      "reward": 0.455729179084301,
      "reward_std": 0.1976660769432783,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 895
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.0938110351562,
      "epoch": 0.3441190590494479,
      "grad_norm": 0.193359375,
      "learning_rate": 8.463840143570762e-07,
      "loss": 0.1137,
      "num_tokens": 156098852.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.19136821990832686,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416567325592,
      "step": 896
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.1458435058594,
      "epoch": 0.3445031204992799,
      "grad_norm": 0.162109375,
      "learning_rate": 8.459295319170859e-07,
      "loss": 0.0779,
      "num_tokens": 156287394.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.19459190173074603,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3593749925494194,
      "step": 897
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.9166870117188,
      "epoch": 0.34488718194911183,
      "grad_norm": 0.1689453125,
      "learning_rate": 8.454745169837238e-07,
      "loss": 0.0237,
      "num_tokens": 156473498.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.15449322666972876,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 898
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.6458740234375,
      "epoch": 0.34527124339894383,
      "grad_norm": 0.189453125,
      "learning_rate": 8.45018970375741e-07,
      "loss": 0.1415,
      "num_tokens": 156653400.0,
      "reward": 0.377604179084301,
      "reward_std": 0.13318732380867004,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 899
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.4271240234375,
      "epoch": 0.3456553048487758,
      "grad_norm": 0.1845703125,
      "learning_rate": 8.445628929128441e-07,
      "loss": 0.0745,
      "num_tokens": 156824969.0,
      "reward": 0.408854179084301,
      "reward_std": 0.10017296997830272,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 900
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1934.1146240234375,
      "epoch": 0.34603936629860776,
      "grad_norm": 0.1552734375,
      "learning_rate": 8.441062854156958e-07,
      "loss": 0.0611,
      "num_tokens": 157025604.0,
      "reward": 0.2942708358168602,
      "reward_std": 0.0470782145857811,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.2942708358168602,
      "step": 901
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.25,
      "epoch": 0.34642342774843976,
      "grad_norm": 0.216796875,
      "learning_rate": 8.436491487059121e-07,
      "loss": 0.1125,
      "num_tokens": 157199500.0,
      "reward": 0.5130208507180214,
      "reward_std": 0.20062683708965778,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 902
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.3125305175781,
      "epoch": 0.3468074891982717,
      "grad_norm": 0.2431640625,
      "learning_rate": 8.431914836060613e-07,
      "loss": 0.1251,
      "num_tokens": 157363682.0,
      "reward": 0.5390625149011612,
      "reward_std": 0.2469298020005226,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 903
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1518.3958740234375,
      "epoch": 0.3471915506481037,
      "grad_norm": 0.25390625,
      "learning_rate": 8.427332909396626e-07,
      "loss": 0.1323,
      "num_tokens": 157522712.0,
      "reward": 0.5546875298023224,
      "reward_std": 0.2518586255609989,
      "rewards/accuracy_reward": 0.15624999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.3542175292969,
      "epoch": 0.3475756120979357,
      "grad_norm": 0.193359375,
      "learning_rate": 8.422745715311842e-07,
      "loss": 0.0527,
      "num_tokens": 157709306.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.19479812122881413,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875074505806,
      "step": 905
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.6146240234375,
      "epoch": 0.34795967354776763,
      "grad_norm": 0.22265625,
      "learning_rate": 8.418153262060427e-07,
      "loss": 0.1182,
      "num_tokens": 157881117.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.1899622566998005,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 906
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.3750610351562,
      "epoch": 0.34834373499759963,
      "grad_norm": 0.21875,
      "learning_rate": 8.413555557906006e-07,
      "loss": 0.0961,
      "num_tokens": 158068689.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.21812857687473297,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 907
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.0104675292969,
      "epoch": 0.34872779644743157,
      "grad_norm": 0.2392578125,
      "learning_rate": 8.408952611121651e-07,
      "loss": 0.1007,
      "num_tokens": 158233922.0,
      "reward": 0.5104166716337204,
      "reward_std": 0.23760602623224258,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 908
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.0521545410156,
      "epoch": 0.34911185789726357,
      "grad_norm": 0.203125,
      "learning_rate": 8.404344429989871e-07,
      "loss": 0.0677,
      "num_tokens": 158392095.0,
      "reward": 0.5677083507180214,
      "reward_std": 0.1947084739804268,
      "rewards/accuracy_reward": 0.14583333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 909
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.1771240234375,
      "epoch": 0.34949591934709556,
      "grad_norm": 0.255859375,
      "learning_rate": 8.399731022802594e-07,
      "loss": 0.1736,
      "num_tokens": 158551408.0,
      "reward": 0.6015625074505806,
      "reward_std": 0.35935673117637634,
      "rewards/accuracy_reward": 0.1562499962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 910
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.2083740234375,
      "epoch": 0.3498799807969275,
      "grad_norm": 0.220703125,
      "learning_rate": 8.395112397861149e-07,
      "loss": 0.0996,
      "num_tokens": 158722708.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.24475315492600203,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 911
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.9688110351562,
      "epoch": 0.3502640422467595,
      "grad_norm": 0.1630859375,
      "learning_rate": 8.390488563476253e-07,
      "loss": 0.0513,
      "num_tokens": 158893081.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.18682263046503067,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 912
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.7812805175781,
      "epoch": 0.35064810369659144,
      "grad_norm": 0.216796875,
      "learning_rate": 8.385859527968003e-07,
      "loss": 0.0619,
      "num_tokens": 159068044.0,
      "reward": 0.518229179084301,
      "reward_std": 0.19954444654285908,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 913
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.5000305175781,
      "epoch": 0.35103216514642344,
      "grad_norm": 0.2158203125,
      "learning_rate": 8.381225299665846e-07,
      "loss": 0.1091,
      "num_tokens": 159233044.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.1531235296279192,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 914
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.8542175292969,
      "epoch": 0.3514162265962554,
      "grad_norm": 0.1796875,
      "learning_rate": 8.37658588690858e-07,
      "loss": 0.0976,
      "num_tokens": 159414686.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.17193811666220427,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.9792175292969,
      "epoch": 0.3518002880460874,
      "grad_norm": 0.267578125,
      "learning_rate": 8.37194129804433e-07,
      "loss": 0.1194,
      "num_tokens": 159569548.0,
      "reward": 0.6223958432674408,
      "reward_std": 0.26387494802474976,
      "rewards/accuracy_reward": 0.15624999534338713,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458283662796,
      "step": 916
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1709.9062805175781,
      "epoch": 0.35218434949591937,
      "grad_norm": 0.20703125,
      "learning_rate": 8.367291541430534e-07,
      "loss": 0.0707,
      "num_tokens": 159747875.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.2042485885322094,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 917
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.1979370117188,
      "epoch": 0.3525684109457513,
      "grad_norm": 0.251953125,
      "learning_rate": 8.362636625433926e-07,
      "loss": 0.131,
      "num_tokens": 159917686.0,
      "reward": 0.510416679084301,
      "reward_std": 0.19002641178667545,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 918
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.1250610351562,
      "epoch": 0.3529524723955833,
      "grad_norm": 0.197265625,
      "learning_rate": 8.357976558430531e-07,
      "loss": 0.1106,
      "num_tokens": 160091626.0,
      "reward": 0.4010416716337204,
      "reward_std": 0.13369719590991735,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906250074505806,
      "step": 919
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1376.0521240234375,
      "epoch": 0.35333653384541525,
      "grad_norm": 0.330078125,
      "learning_rate": 8.353311348805636e-07,
      "loss": 0.2086,
      "num_tokens": 160239359.0,
      "reward": 0.4921875074505806,
      "reward_std": 0.21277756430208683,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 920
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.5937805175781,
      "epoch": 0.35372059529524724,
      "grad_norm": 0.1875,
      "learning_rate": 8.348641004953785e-07,
      "loss": 0.0388,
      "num_tokens": 160410936.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.1759539721533656,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 921
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.8125305175781,
      "epoch": 0.35410465674507924,
      "grad_norm": 0.2265625,
      "learning_rate": 8.34396553527876e-07,
      "loss": 0.0726,
      "num_tokens": 160578606.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.27961915358901024,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 922
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1470.0104370117188,
      "epoch": 0.3544887181949112,
      "grad_norm": 0.1982421875,
      "learning_rate": 8.339284948193564e-07,
      "loss": 0.07,
      "num_tokens": 160731551.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.11315854638814926,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1816.5208740234375,
      "epoch": 0.3548727796447432,
      "grad_norm": 0.193359375,
      "learning_rate": 8.33459925212041e-07,
      "loss": 0.0755,
      "num_tokens": 160921801.0,
      "reward": 0.4375,
      "reward_std": 0.23277632519602776,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.3750305175781,
      "epoch": 0.3552568410945751,
      "grad_norm": 0.2314453125,
      "learning_rate": 8.329908455490708e-07,
      "loss": 0.0718,
      "num_tokens": 161083757.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.14073030836880207,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 925
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1373.3646240234375,
      "epoch": 0.3556409025444071,
      "grad_norm": 0.2109375,
      "learning_rate": 8.325212566745039e-07,
      "loss": 0.0977,
      "num_tokens": 161227360.0,
      "reward": 0.6171875149011612,
      "reward_std": 0.3172619678080082,
      "rewards/accuracy_reward": 0.13541666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4817708358168602,
      "step": 926
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.1875305175781,
      "epoch": 0.35602496399423905,
      "grad_norm": 0.25390625,
      "learning_rate": 8.320511594333151e-07,
      "loss": 0.076,
      "num_tokens": 161387922.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.18630762584507465,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 927
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.2500610351562,
      "epoch": 0.35640902544407105,
      "grad_norm": 0.21875,
      "learning_rate": 8.315805546713938e-07,
      "loss": 0.1157,
      "num_tokens": 161571162.0,
      "reward": 0.4192708358168602,
      "reward_std": 0.15614118613302708,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875074505806,
      "step": 928
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1442.3125305175781,
      "epoch": 0.35679308689390304,
      "grad_norm": 0.220703125,
      "learning_rate": 8.311094432355425e-07,
      "loss": 0.102,
      "num_tokens": 161726296.0,
      "reward": 0.565104179084301,
      "reward_std": 0.25845057144761086,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 929
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.9063110351562,
      "epoch": 0.357177148343735,
      "grad_norm": 0.236328125,
      "learning_rate": 8.306378259734758e-07,
      "loss": 0.1488,
      "num_tokens": 161898039.0,
      "reward": 0.5416666865348816,
      "reward_std": 0.25509550981223583,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 930
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.6250610351562,
      "epoch": 0.357561209793567,
      "grad_norm": 0.208984375,
      "learning_rate": 8.301657037338182e-07,
      "loss": 0.1464,
      "num_tokens": 162075443.0,
      "reward": 0.471354179084301,
      "reward_std": 0.2568073607981205,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 931
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.937515258789,
      "epoch": 0.3579452712433989,
      "grad_norm": 0.1982421875,
      "learning_rate": 8.296930773661029e-07,
      "loss": 0.0629,
      "num_tokens": 162240261.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.1909072231501341,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 932
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.6771240234375,
      "epoch": 0.3583293326932309,
      "grad_norm": 0.193359375,
      "learning_rate": 8.292199477207703e-07,
      "loss": 0.1098,
      "num_tokens": 162424070.0,
      "reward": 0.432291679084301,
      "reward_std": 0.16080001089721918,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 933
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.4479675292969,
      "epoch": 0.3587133941430629,
      "grad_norm": 0.2109375,
      "learning_rate": 8.287463156491662e-07,
      "loss": 0.0622,
      "num_tokens": 162588673.0,
      "reward": 0.5546875149011612,
      "reward_std": 0.2548820972442627,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 934
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1432.1875305175781,
      "epoch": 0.35909745559289485,
      "grad_norm": 0.2373046875,
      "learning_rate": 8.282721820035405e-07,
      "loss": 0.1083,
      "num_tokens": 162742867.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.24546792171895504,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333283662796,
      "step": 935
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.9896240234375,
      "epoch": 0.35948151704272685,
      "grad_norm": 0.1982421875,
      "learning_rate": 8.277975476370458e-07,
      "loss": 0.054,
      "num_tokens": 162914738.0,
      "reward": 0.5729166939854622,
      "reward_std": 0.23716585524380207,
      "rewards/accuracy_reward": 0.17708333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333432674408,
      "step": 936
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.8958740234375,
      "epoch": 0.3598655784925588,
      "grad_norm": 0.2421875,
      "learning_rate": 8.273224134037357e-07,
      "loss": 0.1447,
      "num_tokens": 163072568.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.23627649992704391,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 937
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1670.6979370117188,
      "epoch": 0.3602496399423908,
      "grad_norm": 0.1884765625,
      "learning_rate": 8.26846780158563e-07,
      "loss": 0.1347,
      "num_tokens": 163247651.0,
      "reward": 0.502604179084301,
      "reward_std": 0.24599530128762126,
      "rewards/accuracy_reward": 0.1249999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 938
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1609.5625610351562,
      "epoch": 0.3606337013922228,
      "grad_norm": 0.2490234375,
      "learning_rate": 8.263706487573788e-07,
      "loss": 0.1436,
      "num_tokens": 163417905.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.1847914420068264,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 939
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.0000305175781,
      "epoch": 0.3610177628420547,
      "grad_norm": 0.259765625,
      "learning_rate": 8.258940200569302e-07,
      "loss": 0.0991,
      "num_tokens": 163592401.0,
      "reward": 0.471354179084301,
      "reward_std": 0.24681582301855087,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.388020820915699,
      "step": 940
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1470.5938110351562,
      "epoch": 0.3614018242918867,
      "grad_norm": 0.26953125,
      "learning_rate": 8.254168949148594e-07,
      "loss": 0.1277,
      "num_tokens": 163748266.0,
      "reward": 0.432291679084301,
      "reward_std": 0.1474077757447958,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 941
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1398.8959045410156,
      "epoch": 0.36178588574171866,
      "grad_norm": 0.251953125,
      "learning_rate": 8.249392741897016e-07,
      "loss": 0.0693,
      "num_tokens": 163895016.0,
      "reward": 0.5677083432674408,
      "reward_std": 0.22778241336345673,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583358168602,
      "step": 942
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.3229370117188,
      "epoch": 0.36216994719155066,
      "grad_norm": 0.2109375,
      "learning_rate": 8.244611587408841e-07,
      "loss": 0.1188,
      "num_tokens": 164064439.0,
      "reward": 0.549479179084301,
      "reward_std": 0.2995961718261242,
      "rewards/accuracy_reward": 0.15625000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 943
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1673.9792175292969,
      "epoch": 0.3625540086413826,
      "grad_norm": 0.240234375,
      "learning_rate": 8.23982549428724e-07,
      "loss": 0.0902,
      "num_tokens": 164237189.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.24719548225402832,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 944
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1475.0208435058594,
      "epoch": 0.3629380700912146,
      "grad_norm": 0.27734375,
      "learning_rate": 8.235034471144277e-07,
      "loss": 0.1107,
      "num_tokens": 164392695.0,
      "reward": 0.4921875074505806,
      "reward_std": 0.22526633366942406,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375074505806,
      "step": 945
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.8542175292969,
      "epoch": 0.3633221315410466,
      "grad_norm": 0.1845703125,
      "learning_rate": 8.23023852660088e-07,
      "loss": 0.0545,
      "num_tokens": 164562073.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.15795482322573662,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 946
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1691.3229370117188,
      "epoch": 0.36370619299087853,
      "grad_norm": 0.197265625,
      "learning_rate": 8.225437669286838e-07,
      "loss": 0.1167,
      "num_tokens": 164740568.0,
      "reward": 0.4322916865348816,
      "reward_std": 0.18003157898783684,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 947
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.0104675292969,
      "epoch": 0.3640902544407105,
      "grad_norm": 0.1953125,
      "learning_rate": 8.220631907840774e-07,
      "loss": 0.0769,
      "num_tokens": 164915865.0,
      "reward": 0.6015625149011612,
      "reward_std": 0.27543993666768074,
      "rewards/accuracy_reward": 0.23958333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 948
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.7291870117188,
      "epoch": 0.36447431589054247,
      "grad_norm": 0.193359375,
      "learning_rate": 8.215821250910141e-07,
      "loss": 0.0805,
      "num_tokens": 165073559.0,
      "reward": 0.5312500074505806,
      "reward_std": 0.17331242188811302,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 949
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.4583740234375,
      "epoch": 0.36485837734037446,
      "grad_norm": 0.205078125,
      "learning_rate": 8.211005707151198e-07,
      "loss": 0.1244,
      "num_tokens": 165255835.0,
      "reward": 0.401041679084301,
      "reward_std": 0.16266683489084244,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 950
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1704.3125610351562,
      "epoch": 0.36524243879020646,
      "grad_norm": 0.21875,
      "learning_rate": 8.206185285228999e-07,
      "loss": 0.0922,
      "num_tokens": 165434633.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.16059836000204086,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 951
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1858.4479370117188,
      "epoch": 0.3656265002400384,
      "grad_norm": 0.181640625,
      "learning_rate": 8.201359993817374e-07,
      "loss": 0.1123,
      "num_tokens": 165626212.0,
      "reward": 0.3750000074505806,
      "reward_std": 0.15869775786995888,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333358168602,
      "step": 952
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1454.4063110351562,
      "epoch": 0.3660105616898704,
      "grad_norm": 0.26953125,
      "learning_rate": 8.19652984159892e-07,
      "loss": 0.0776,
      "num_tokens": 165779899.0,
      "reward": 0.6041666865348816,
      "reward_std": 0.3088437393307686,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.4895935058594,
      "epoch": 0.36639462313970234,
      "grad_norm": 0.2119140625,
      "learning_rate": 8.19169483726497e-07,
      "loss": 0.1238,
      "num_tokens": 165959938.0,
      "reward": 0.416666679084301,
      "reward_std": 0.17648079618811607,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 954
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.3959045410156,
      "epoch": 0.36677868458953433,
      "grad_norm": 0.255859375,
      "learning_rate": 8.186854989515598e-07,
      "loss": 0.0819,
      "num_tokens": 166120112.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.25014741718769073,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 955
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1823.5833435058594,
      "epoch": 0.3671627460393663,
      "grad_norm": 0.2041015625,
      "learning_rate": 8.182010307059591e-07,
      "loss": 0.1077,
      "num_tokens": 166311616.0,
      "reward": 0.473958358168602,
      "reward_std": 0.3175192326307297,
      "rewards/accuracy_reward": 0.11458333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 956
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.6041870117188,
      "epoch": 0.36754680748919827,
      "grad_norm": 0.2099609375,
      "learning_rate": 8.177160798614431e-07,
      "loss": 0.0888,
      "num_tokens": 166480882.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.12497659958899021,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1516.1979675292969,
      "epoch": 0.36793086893903026,
      "grad_norm": 0.267578125,
      "learning_rate": 8.172306472906288e-07,
      "loss": 0.2107,
      "num_tokens": 166643165.0,
      "reward": 0.5546875149011612,
      "reward_std": 0.30640842020511627,
      "rewards/accuracy_reward": 0.1249999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 958
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1808.8125610351562,
      "epoch": 0.3683149303888622,
      "grad_norm": 0.189453125,
      "learning_rate": 8.167447338670001e-07,
      "loss": 0.0805,
      "num_tokens": 166831675.0,
      "reward": 0.4921875298023224,
      "reward_std": 0.19989371486008167,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 959
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.8958435058594,
      "epoch": 0.3686989918386942,
      "grad_norm": 0.2421875,
      "learning_rate": 8.162583404649056e-07,
      "loss": 0.1197,
      "num_tokens": 166989929.0,
      "reward": 0.5000000074505806,
      "reward_std": 0.2041627075523138,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 960
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.6771240234375,
      "epoch": 0.36908305328852614,
      "grad_norm": 0.263671875,
      "learning_rate": 8.157714679595583e-07,
      "loss": 0.109,
      "num_tokens": 167156866.0,
      "reward": 0.455729179084301,
      "reward_std": 0.16508889384567738,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 961
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1832.4583740234375,
      "epoch": 0.36946711473835814,
      "grad_norm": 0.181640625,
      "learning_rate": 8.152841172270326e-07,
      "loss": 0.0617,
      "num_tokens": 167349078.0,
      "reward": 0.4531250074505806,
      "reward_std": 0.15327804442495108,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583283662796,
      "step": 962
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.4687805175781,
      "epoch": 0.36985117618819013,
      "grad_norm": 0.2275390625,
      "learning_rate": 8.14796289144264e-07,
      "loss": 0.1157,
      "num_tokens": 167505307.0,
      "reward": 0.541666679084301,
      "reward_std": 0.24905062839388847,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 963
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.1354675292969,
      "epoch": 0.3702352376380221,
      "grad_norm": 0.2236328125,
      "learning_rate": 8.143079845890465e-07,
      "loss": 0.1054,
      "num_tokens": 167662360.0,
      "reward": 0.5234375298023224,
      "reward_std": 0.22442393004894257,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 964
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.5729675292969,
      "epoch": 0.37061929908785407,
      "grad_norm": 0.16796875,
      "learning_rate": 8.138192044400319e-07,
      "loss": 0.0857,
      "num_tokens": 167847383.0,
      "reward": 0.4505208507180214,
      "reward_std": 0.17864275351166725,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 965
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.9896240234375,
      "epoch": 0.371003360537686,
      "grad_norm": 0.1669921875,
      "learning_rate": 8.13329949576727e-07,
      "loss": 0.051,
      "num_tokens": 168012910.0,
      "reward": 0.455729179084301,
      "reward_std": 0.08084372617304325,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 966
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1413.1250305175781,
      "epoch": 0.371387421987518,
      "grad_norm": 0.2490234375,
      "learning_rate": 8.128402208794939e-07,
      "loss": 0.0964,
      "num_tokens": 168161778.0,
      "reward": 0.6484375298023224,
      "reward_std": 0.2873328886926174,
      "rewards/accuracy_reward": 0.21875000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541641831398,
      "step": 967
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.7708740234375,
      "epoch": 0.37177148343735,
      "grad_norm": 0.2138671875,
      "learning_rate": 8.123500192295463e-07,
      "loss": 0.0356,
      "num_tokens": 168329476.0,
      "reward": 0.5130208507180214,
      "reward_std": 0.20875925943255424,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 968
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.1354370117188,
      "epoch": 0.37215554488718194,
      "grad_norm": 0.203125,
      "learning_rate": 8.118593455089496e-07,
      "loss": 0.0982,
      "num_tokens": 168504105.0,
      "reward": 0.5078125223517418,
      "reward_std": 0.2632557041943073,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 969
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.7083740234375,
      "epoch": 0.37253960633701394,
      "grad_norm": 0.2392578125,
      "learning_rate": 8.113682006006183e-07,
      "loss": 0.1271,
      "num_tokens": 168670117.0,
      "reward": 0.5572916865348816,
      "reward_std": 0.2722702771425247,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 970
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.4583740234375,
      "epoch": 0.3729236677868459,
      "grad_norm": 0.24609375,
      "learning_rate": 8.108765853883151e-07,
      "loss": 0.1134,
      "num_tokens": 168842897.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.16801253519952297,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 971
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1834.4479370117188,
      "epoch": 0.3733077292366779,
      "grad_norm": 0.197265625,
      "learning_rate": 8.103845007566482e-07,
      "loss": 0.0915,
      "num_tokens": 169032780.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.25765473395586014,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 972
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.0312805175781,
      "epoch": 0.3736917906865098,
      "grad_norm": 0.166015625,
      "learning_rate": 8.098919475910714e-07,
      "loss": 0.1051,
      "num_tokens": 169212431.0,
      "reward": 0.3723958432674408,
      "reward_std": 0.13100438937544823,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 973
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.3646240234375,
      "epoch": 0.3740758521363418,
      "grad_norm": 0.23828125,
      "learning_rate": 8.09398926777881e-07,
      "loss": 0.031,
      "num_tokens": 169390634.0,
      "reward": 0.489583358168602,
      "reward_std": 0.1605494050309062,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 974
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1824.1875305175781,
      "epoch": 0.3744599135861738,
      "grad_norm": 0.17578125,
      "learning_rate": 8.08905439204215e-07,
      "loss": 0.0629,
      "num_tokens": 169578436.0,
      "reward": 0.3932291716337204,
      "reward_std": 0.12620627507567406,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458283662796,
      "step": 975
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.4271240234375,
      "epoch": 0.37484397503600575,
      "grad_norm": 0.25390625,
      "learning_rate": 8.084114857580512e-07,
      "loss": 0.0888,
      "num_tokens": 169752181.0,
      "reward": 0.4401041865348816,
      "reward_std": 0.20128452964127064,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 976
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.6146240234375,
      "epoch": 0.37522803648583775,
      "grad_norm": 0.193359375,
      "learning_rate": 8.079170673282057e-07,
      "loss": 0.1516,
      "num_tokens": 169939808.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.24709966778755188,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 977
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1833.0833740234375,
      "epoch": 0.3756120979356697,
      "grad_norm": 0.1865234375,
      "learning_rate": 8.074221848043313e-07,
      "loss": 0.0993,
      "num_tokens": 170131064.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.21229775622487068,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 978
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.5417175292969,
      "epoch": 0.3759961593855017,
      "grad_norm": 0.1982421875,
      "learning_rate": 8.069268390769158e-07,
      "loss": 0.0493,
      "num_tokens": 170299844.0,
      "reward": 0.541666679084301,
      "reward_std": 0.23747742176055908,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 979
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.1042175292969,
      "epoch": 0.3763802208353337,
      "grad_norm": 0.2080078125,
      "learning_rate": 8.064310310372806e-07,
      "loss": 0.0795,
      "num_tokens": 170477150.0,
      "reward": 0.596354179084301,
      "reward_std": 0.303592074662447,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 980
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.5313110351562,
      "epoch": 0.3767642822851656,
      "grad_norm": 0.2060546875,
      "learning_rate": 8.059347615775792e-07,
      "loss": 0.0846,
      "num_tokens": 170645849.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.22900167852640152,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1677.5208740234375,
      "epoch": 0.3771483437349976,
      "grad_norm": 0.201171875,
      "learning_rate": 8.054380315907949e-07,
      "loss": 0.0859,
      "num_tokens": 170820947.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.1060713455080986,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 982
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1414.4479675292969,
      "epoch": 0.37753240518482956,
      "grad_norm": 0.2255859375,
      "learning_rate": 8.049408419707398e-07,
      "loss": 0.137,
      "num_tokens": 170972670.0,
      "reward": 0.630208358168602,
      "reward_std": 0.24377158284187317,
      "rewards/accuracy_reward": 0.1562500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583283662796,
      "step": 983
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.9479675292969,
      "epoch": 0.37791646663466155,
      "grad_norm": 0.1806640625,
      "learning_rate": 8.044431936120536e-07,
      "loss": 0.0501,
      "num_tokens": 171145369.0,
      "reward": 0.4114583507180214,
      "reward_std": 0.11954757571220398,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 984
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1385.7604370117188,
      "epoch": 0.3783005280844935,
      "grad_norm": 0.2734375,
      "learning_rate": 8.039450874102004e-07,
      "loss": 0.0529,
      "num_tokens": 171292250.0,
      "reward": 0.59375,
      "reward_std": 0.27067236602306366,
      "rewards/accuracy_reward": 0.1458333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 985
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.291748046875,
      "epoch": 0.3786845895343255,
      "grad_norm": 0.251953125,
      "learning_rate": 8.034465242614693e-07,
      "loss": 0.0666,
      "num_tokens": 171448022.0,
      "reward": 0.5937500223517418,
      "reward_std": 0.2996930480003357,
      "rewards/accuracy_reward": 0.13541666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 986
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1335.843765258789,
      "epoch": 0.3790686509841575,
      "grad_norm": 0.203125,
      "learning_rate": 8.029475050629707e-07,
      "loss": 0.06,
      "num_tokens": 171589631.0,
      "reward": 0.5130208358168602,
      "reward_std": 0.16375793097540736,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 987
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.9271240234375,
      "epoch": 0.3794527124339894,
      "grad_norm": 0.1962890625,
      "learning_rate": 8.024480307126361e-07,
      "loss": 0.124,
      "num_tokens": 171766984.0,
      "reward": 0.447916679084301,
      "reward_std": 0.23162922263145447,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3750000074505806,
      "step": 988
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1509.1979675292969,
      "epoch": 0.3798367738838214,
      "grad_norm": 0.2001953125,
      "learning_rate": 8.01948102109216e-07,
      "loss": 0.1574,
      "num_tokens": 171925819.0,
      "reward": 0.5546875223517418,
      "reward_std": 0.2817341983318329,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875074505806,
      "step": 989
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.1250610351562,
      "epoch": 0.38022083533365336,
      "grad_norm": 0.2158203125,
      "learning_rate": 8.014477201522781e-07,
      "loss": 0.0781,
      "num_tokens": 172096503.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.2359677325002849,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 990
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1722.8020935058594,
      "epoch": 0.38060489678348536,
      "grad_norm": 0.177734375,
      "learning_rate": 8.009468857422058e-07,
      "loss": 0.1177,
      "num_tokens": 172276876.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.16585673484951258,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 991
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.6771545410156,
      "epoch": 0.38098895823331735,
      "grad_norm": 0.201171875,
      "learning_rate": 8.004455997801969e-07,
      "loss": 0.0492,
      "num_tokens": 172449261.0,
      "reward": 0.4869791716337204,
      "reward_std": 0.17262177914381027,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 992
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.8125305175781,
      "epoch": 0.3813730196831493,
      "grad_norm": 0.23046875,
      "learning_rate": 7.999438631682615e-07,
      "loss": 0.1017,
      "num_tokens": 172608603.0,
      "reward": 0.5755208432674408,
      "reward_std": 0.2527368515729904,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 993
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.5104370117188,
      "epoch": 0.3817570811329813,
      "grad_norm": 0.2294921875,
      "learning_rate": 7.994416768092209e-07,
      "loss": 0.1109,
      "num_tokens": 172776292.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.24406202137470245,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 994
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.2188110351562,
      "epoch": 0.38214114258281323,
      "grad_norm": 0.2099609375,
      "learning_rate": 7.989390416067051e-07,
      "loss": 0.1039,
      "num_tokens": 172957113.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.2817212678492069,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 995
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.0000610351562,
      "epoch": 0.3825252040326452,
      "grad_norm": 0.22265625,
      "learning_rate": 7.984359584651522e-07,
      "loss": 0.1361,
      "num_tokens": 173119065.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.2854057550430298,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 996
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1559.3021392822266,
      "epoch": 0.3829092654824772,
      "grad_norm": 0.2734375,
      "learning_rate": 7.979324282898065e-07,
      "loss": 0.0857,
      "num_tokens": 173282710.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.20482757315039635,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 997
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1648.8438110351562,
      "epoch": 0.38329332693230916,
      "grad_norm": 0.2041015625,
      "learning_rate": 7.974284519867163e-07,
      "loss": 0.0707,
      "num_tokens": 173458479.0,
      "reward": 0.4817708358168602,
      "reward_std": 0.22445886954665184,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 998
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.4583740234375,
      "epoch": 0.38367738838214116,
      "grad_norm": 0.2138671875,
      "learning_rate": 7.969240304627325e-07,
      "loss": 0.11,
      "num_tokens": 173629075.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.20683813467621803,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 999
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.0625610351562,
      "epoch": 0.3840614498319731,
      "grad_norm": 0.228515625,
      "learning_rate": 7.964191646255076e-07,
      "loss": 0.1172,
      "num_tokens": 173800769.0,
      "reward": 0.4843750223517418,
      "reward_std": 0.1925459410995245,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.0417175292969,
      "epoch": 0.3844455112818051,
      "grad_norm": 0.208984375,
      "learning_rate": 7.959138553834934e-07,
      "loss": 0.0522,
      "num_tokens": 173974973.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.12239916808903217,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 1001
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1522.5000610351562,
      "epoch": 0.38482957273163704,
      "grad_norm": 0.2060546875,
      "learning_rate": 7.954081036459394e-07,
      "loss": 0.0551,
      "num_tokens": 174134877.0,
      "reward": 0.4739583507180214,
      "reward_std": 0.11272067204117775,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 1002
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1404.1667175292969,
      "epoch": 0.38521363418146903,
      "grad_norm": 0.203125,
      "learning_rate": 7.949019103228916e-07,
      "loss": 0.0393,
      "num_tokens": 174282125.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.17970804683864117,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4791666716337204,
      "step": 1003
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.4687805175781,
      "epoch": 0.38559769563130103,
      "grad_norm": 0.2177734375,
      "learning_rate": 7.943952763251903e-07,
      "loss": 0.0412,
      "num_tokens": 174450186.0,
      "reward": 0.4973958358168602,
      "reward_std": 0.15555139258503914,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1004
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.1875610351562,
      "epoch": 0.38598175708113297,
      "grad_norm": 0.2080078125,
      "learning_rate": 7.938882025644692e-07,
      "loss": 0.1066,
      "num_tokens": 174622324.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.22329498641192913,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1005
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.6770935058594,
      "epoch": 0.38636581853096497,
      "grad_norm": 0.1923828125,
      "learning_rate": 7.933806899531528e-07,
      "loss": 0.0296,
      "num_tokens": 174782989.0,
      "reward": 0.5000000223517418,
      "reward_std": 0.18190074246376753,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1006
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.8646545410156,
      "epoch": 0.3867498799807969,
      "grad_norm": 0.236328125,
      "learning_rate": 7.928727394044554e-07,
      "loss": 0.0987,
      "num_tokens": 174950928.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.17279627732932568,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1007
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.1250610351562,
      "epoch": 0.3871339414306289,
      "grad_norm": 0.2314453125,
      "learning_rate": 7.923643518323796e-07,
      "loss": 0.099,
      "num_tokens": 175140868.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.1741786114871502,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1008
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.8333740234375,
      "epoch": 0.3875180028804609,
      "grad_norm": 0.2158203125,
      "learning_rate": 7.918555281517141e-07,
      "loss": 0.0893,
      "num_tokens": 175314028.0,
      "reward": 0.4869791716337204,
      "reward_std": 0.19462630711495876,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 1009
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.7187805175781,
      "epoch": 0.38790206433029284,
      "grad_norm": 0.189453125,
      "learning_rate": 7.913462692780321e-07,
      "loss": 0.1197,
      "num_tokens": 175488161.0,
      "reward": 0.463541679084301,
      "reward_std": 0.2617034278810024,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1010
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.8854675292969,
      "epoch": 0.38828612578012484,
      "grad_norm": 0.2060546875,
      "learning_rate": 7.908365761276905e-07,
      "loss": 0.1013,
      "num_tokens": 175665494.0,
      "reward": 0.541666679084301,
      "reward_std": 0.20646862965077162,
      "rewards/accuracy_reward": 0.1770833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 1011
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1871.5104370117188,
      "epoch": 0.3886701872299568,
      "grad_norm": 0.1689453125,
      "learning_rate": 7.903264496178274e-07,
      "loss": 0.0813,
      "num_tokens": 175859239.0,
      "reward": 0.322916679084301,
      "reward_std": 0.09265054017305374,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3229166641831398,
      "step": 1012
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1817.3021545410156,
      "epoch": 0.38905424867978877,
      "grad_norm": 0.189453125,
      "learning_rate": 7.898158906663605e-07,
      "loss": 0.1063,
      "num_tokens": 176050100.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.21685649454593658,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666716337204,
      "step": 1013
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1515.8646545410156,
      "epoch": 0.3894383101296207,
      "grad_norm": 0.2421875,
      "learning_rate": 7.893049001919855e-07,
      "loss": 0.0806,
      "num_tokens": 176209607.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.215045977383852,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1014
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1434.7291870117188,
      "epoch": 0.3898223715794527,
      "grad_norm": 0.205078125,
      "learning_rate": 7.887934791141752e-07,
      "loss": 0.0639,
      "num_tokens": 176362501.0,
      "reward": 0.5520833507180214,
      "reward_std": 0.22092893719673157,
      "rewards/accuracy_reward": 0.12500000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1015
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.3854370117188,
      "epoch": 0.3902064330292847,
      "grad_norm": 0.2041015625,
      "learning_rate": 7.882816283531766e-07,
      "loss": 0.099,
      "num_tokens": 176531618.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.16592027433216572,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041716337204,
      "step": 1016
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1411.8437805175781,
      "epoch": 0.39059049447911665,
      "grad_norm": 0.234375,
      "learning_rate": 7.8776934883001e-07,
      "loss": 0.0423,
      "num_tokens": 176680003.0,
      "reward": 0.5156250298023224,
      "reward_std": 0.1747160702943802,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5052083283662796,
      "step": 1017
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1425.2812805175781,
      "epoch": 0.39097455592894864,
      "grad_norm": 0.26171875,
      "learning_rate": 7.872566414664674e-07,
      "loss": 0.1113,
      "num_tokens": 176830222.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.23935507237911224,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609374925494194,
      "step": 1018
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1446.0209045410156,
      "epoch": 0.3913586173787806,
      "grad_norm": 0.25390625,
      "learning_rate": 7.86743507185111e-07,
      "loss": 0.139,
      "num_tokens": 176984840.0,
      "reward": 0.5651041865348816,
      "reward_std": 0.24134209007024765,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 1019
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.8021392822266,
      "epoch": 0.3917426788286126,
      "grad_norm": 0.2373046875,
      "learning_rate": 7.862299469092702e-07,
      "loss": 0.1419,
      "num_tokens": 177148669.0,
      "reward": 0.5651041716337204,
      "reward_std": 0.2269229143857956,
      "rewards/accuracy_reward": 0.1249999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 1020
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.9584045410156,
      "epoch": 0.3921267402784446,
      "grad_norm": 0.2314453125,
      "learning_rate": 7.857159615630417e-07,
      "loss": 0.1026,
      "num_tokens": 177313657.0,
      "reward": 0.4687500223517418,
      "reward_std": 0.23526381701231003,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 1021
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.9062805175781,
      "epoch": 0.3925108017282765,
      "grad_norm": 0.2177734375,
      "learning_rate": 7.852015520712866e-07,
      "loss": 0.0555,
      "num_tokens": 177480632.0,
      "reward": 0.494791679084301,
      "reward_std": 0.16550043411552906,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 1022
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.3125610351562,
      "epoch": 0.3928948631781085,
      "grad_norm": 0.19921875,
      "learning_rate": 7.8468671935963e-07,
      "loss": 0.0829,
      "num_tokens": 177665382.0,
      "reward": 0.502604179084301,
      "reward_std": 0.2733879014849663,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 1023
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.2709045410156,
      "epoch": 0.39327892462794045,
      "grad_norm": 0.2216796875,
      "learning_rate": 7.841714643544577e-07,
      "loss": 0.0704,
      "num_tokens": 177834120.0,
      "reward": 0.5703125074505806,
      "reward_std": 0.2802669256925583,
      "rewards/accuracy_reward": 0.15625000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1024
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.7708740234375,
      "epoch": 0.39366298607777245,
      "grad_norm": 0.21875,
      "learning_rate": 7.836557879829155e-07,
      "loss": 0.0936,
      "num_tokens": 178017658.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.15950089506804943,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 1025
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.2187805175781,
      "epoch": 0.39404704752760444,
      "grad_norm": 0.2578125,
      "learning_rate": 7.83139691172908e-07,
      "loss": 0.127,
      "num_tokens": 178186863.0,
      "reward": 0.5338541939854622,
      "reward_std": 0.27382319048047066,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 1026
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.4167175292969,
      "epoch": 0.3944311089774364,
      "grad_norm": 0.1962890625,
      "learning_rate": 7.826231748530956e-07,
      "loss": 0.125,
      "num_tokens": 178371959.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.15049160225316882,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1027
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.4688110351562,
      "epoch": 0.3948151704272684,
      "grad_norm": 0.22265625,
      "learning_rate": 7.821062399528939e-07,
      "loss": 0.0539,
      "num_tokens": 178532868.0,
      "reward": 0.510416679084301,
      "reward_std": 0.2010174971073866,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 1028
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1449.5000305175781,
      "epoch": 0.3951992318771003,
      "grad_norm": 0.322265625,
      "learning_rate": 7.815888874024718e-07,
      "loss": 0.1599,
      "num_tokens": 178685892.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.23866023123264313,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 1029
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1567.0938110351562,
      "epoch": 0.3955832933269323,
      "grad_norm": 0.2294921875,
      "learning_rate": 7.810711181327495e-07,
      "loss": 0.13,
      "num_tokens": 178850701.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.2560054212808609,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1030
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1809.8229675292969,
      "epoch": 0.39596735477676426,
      "grad_norm": 0.21484375,
      "learning_rate": 7.805529330753972e-07,
      "loss": 0.0627,
      "num_tokens": 179038948.0,
      "reward": 0.369791679084301,
      "reward_std": 0.07721341168507934,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1031
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.2187805175781,
      "epoch": 0.39635141622659625,
      "grad_norm": 0.1533203125,
      "learning_rate": 7.800343331628331e-07,
      "loss": 0.0335,
      "num_tokens": 179228089.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.11797146499156952,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833432674408,
      "step": 1032
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1667.6979675292969,
      "epoch": 0.39673547767642825,
      "grad_norm": 0.2216796875,
      "learning_rate": 7.79515319328222e-07,
      "loss": 0.1296,
      "num_tokens": 179401836.0,
      "reward": 0.4609375298023224,
      "reward_std": 0.19334934279322624,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 1033
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.0208740234375,
      "epoch": 0.3971195391262602,
      "grad_norm": 0.1953125,
      "learning_rate": 7.789958925054735e-07,
      "loss": 0.0614,
      "num_tokens": 179581374.0,
      "reward": 0.447916679084301,
      "reward_std": 0.19606922566890717,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 1034
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1702.7917175292969,
      "epoch": 0.3975036005760922,
      "grad_norm": 0.2314453125,
      "learning_rate": 7.784760536292402e-07,
      "loss": 0.0747,
      "num_tokens": 179758994.0,
      "reward": 0.3671875149011612,
      "reward_std": 0.11696073738858104,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 1035
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1709.4479370117188,
      "epoch": 0.3978876620259241,
      "grad_norm": 0.224609375,
      "learning_rate": 7.779558036349165e-07,
      "loss": 0.101,
      "num_tokens": 179940309.0,
      "reward": 0.4218750223517418,
      "reward_std": 0.14766379352658987,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 1036
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1489.2916870117188,
      "epoch": 0.3982717234757561,
      "grad_norm": 0.2451171875,
      "learning_rate": 7.774351434586361e-07,
      "loss": 0.1053,
      "num_tokens": 180095281.0,
      "reward": 0.6197917014360428,
      "reward_std": 0.3636337071657181,
      "rewards/accuracy_reward": 0.1770833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1037
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.6146545410156,
      "epoch": 0.3986557849255881,
      "grad_norm": 0.208984375,
      "learning_rate": 7.769140740372713e-07,
      "loss": 0.119,
      "num_tokens": 180258908.0,
      "reward": 0.575520858168602,
      "reward_std": 0.20739070512354374,
      "rewards/accuracy_reward": 0.1458333283662796,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1038
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1458.6042175292969,
      "epoch": 0.39903984637542006,
      "grad_norm": 0.2373046875,
      "learning_rate": 7.763925963084301e-07,
      "loss": 0.0771,
      "num_tokens": 180410590.0,
      "reward": 0.606770858168602,
      "reward_std": 0.23654454946517944,
      "rewards/accuracy_reward": 0.15624999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 1039
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.3542175292969,
      "epoch": 0.39942390782525206,
      "grad_norm": 0.2353515625,
      "learning_rate": 7.758707112104558e-07,
      "loss": 0.0842,
      "num_tokens": 180571536.0,
      "reward": 0.5312500298023224,
      "reward_std": 0.26339912973344326,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4374999925494194,
      "step": 1040
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1892.7187805175781,
      "epoch": 0.399807969275084,
      "grad_norm": 0.17578125,
      "learning_rate": 7.753484196824246e-07,
      "loss": 0.0877,
      "num_tokens": 180767061.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.23019808903336525,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 1041
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.1667175292969,
      "epoch": 0.400192030724916,
      "grad_norm": 0.2412109375,
      "learning_rate": 7.748257226641436e-07,
      "loss": 0.105,
      "num_tokens": 180933357.0,
      "reward": 0.5546875149011612,
      "reward_std": 0.21262122504413128,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1042
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.5625610351562,
      "epoch": 0.40057609217474793,
      "grad_norm": 0.2138671875,
      "learning_rate": 7.743026210961504e-07,
      "loss": 0.1075,
      "num_tokens": 181120299.0,
      "reward": 0.3750000074505806,
      "reward_std": 0.16547270771116018,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 1043
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.8542175292969,
      "epoch": 0.40096015362457993,
      "grad_norm": 0.251953125,
      "learning_rate": 7.737791159197096e-07,
      "loss": 0.1428,
      "num_tokens": 181292765.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.23939158394932747,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1044
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.1250610351562,
      "epoch": 0.4013442150744119,
      "grad_norm": 0.205078125,
      "learning_rate": 7.732552080768127e-07,
      "loss": 0.0666,
      "num_tokens": 181466345.0,
      "reward": 0.479166679084301,
      "reward_std": 0.1752810124307871,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1045
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1435.9583740234375,
      "epoch": 0.40172827652424387,
      "grad_norm": 0.25,
      "learning_rate": 7.727308985101753e-07,
      "loss": 0.1346,
      "num_tokens": 181617053.0,
      "reward": 0.5781250149011612,
      "reward_std": 0.2735988050699234,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583358168602,
      "step": 1046
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.8854675292969,
      "epoch": 0.40211233797407586,
      "grad_norm": 0.2177734375,
      "learning_rate": 7.722061881632365e-07,
      "loss": 0.0456,
      "num_tokens": 181782338.0,
      "reward": 0.5781250223517418,
      "reward_std": 0.27610624209046364,
      "rewards/accuracy_reward": 0.13541666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 1047
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1699.6041870117188,
      "epoch": 0.4024963994239078,
      "grad_norm": 0.2431640625,
      "learning_rate": 7.716810779801555e-07,
      "loss": 0.132,
      "num_tokens": 181959748.0,
      "reward": 0.458333358168602,
      "reward_std": 0.21895207464694977,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 1048
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1531.1667175292969,
      "epoch": 0.4028804608737398,
      "grad_norm": 0.244140625,
      "learning_rate": 7.711555689058124e-07,
      "loss": 0.1041,
      "num_tokens": 182119932.0,
      "reward": 0.533854179084301,
      "reward_std": 0.2023411002010107,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 1049
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.0208740234375,
      "epoch": 0.4032645223235718,
      "grad_norm": 0.25,
      "learning_rate": 7.706296618858034e-07,
      "loss": 0.0867,
      "num_tokens": 182289734.0,
      "reward": 0.510416679084301,
      "reward_std": 0.2609420493245125,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 1050
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1495.2396240234375,
      "epoch": 0.40364858377340374,
      "grad_norm": 0.21875,
      "learning_rate": 7.701033578664425e-07,
      "loss": 0.1229,
      "num_tokens": 182445341.0,
      "reward": 0.5937500223517418,
      "reward_std": 0.293410949409008,
      "rewards/accuracy_reward": 0.16145833302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333283662796,
      "step": 1051
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1419.2916870117188,
      "epoch": 0.40403264522323573,
      "grad_norm": 0.25390625,
      "learning_rate": 7.695766577947565e-07,
      "loss": 0.1005,
      "num_tokens": 182595249.0,
      "reward": 0.567708358168602,
      "reward_std": 0.2746334783732891,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 1052
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.90625,
      "epoch": 0.40441670667306767,
      "grad_norm": 0.16796875,
      "learning_rate": 7.690495626184861e-07,
      "loss": 0.0555,
      "num_tokens": 182772672.0,
      "reward": 0.432291679084301,
      "reward_std": 0.1919987229630351,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 1053
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.4896240234375,
      "epoch": 0.40480076812289967,
      "grad_norm": 0.212890625,
      "learning_rate": 7.685220732860818e-07,
      "loss": 0.1046,
      "num_tokens": 182951543.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.17225927766412497,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1054
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.3333740234375,
      "epoch": 0.40518482957273166,
      "grad_norm": 0.16015625,
      "learning_rate": 7.679941907467044e-07,
      "loss": 0.0773,
      "num_tokens": 183133271.0,
      "reward": 0.393229179084301,
      "reward_std": 0.12034751800820231,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 1055
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1506.4791870117188,
      "epoch": 0.4055688910225636,
      "grad_norm": 0.216796875,
      "learning_rate": 7.674659159502216e-07,
      "loss": 0.1039,
      "num_tokens": 183293141.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.2522929832339287,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125074505806,
      "step": 1056
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.4792175292969,
      "epoch": 0.4059529524723956,
      "grad_norm": 0.2060546875,
      "learning_rate": 7.669372498472072e-07,
      "loss": 0.1086,
      "num_tokens": 183478675.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.2869963627308607,
      "rewards/accuracy_reward": 0.14583333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 1057
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.7916870117188,
      "epoch": 0.40633701392222754,
      "grad_norm": 0.2021484375,
      "learning_rate": 7.66408193388939e-07,
      "loss": 0.1343,
      "num_tokens": 183670599.0,
      "reward": 0.3307291716337204,
      "reward_std": 0.11002940405160189,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291716337204,
      "step": 1058
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.4479675292969,
      "epoch": 0.40672107537205954,
      "grad_norm": 0.22265625,
      "learning_rate": 7.658787475273967e-07,
      "loss": 0.1271,
      "num_tokens": 183838098.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.2530040852725506,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 1059
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1398.6979675292969,
      "epoch": 0.4071051368218915,
      "grad_norm": 0.2451171875,
      "learning_rate": 7.653489132152619e-07,
      "loss": 0.1309,
      "num_tokens": 183986725.0,
      "reward": 0.6223958358168602,
      "reward_std": 0.27296826243400574,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4869791716337204,
      "step": 1060
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.1875305175781,
      "epoch": 0.4074891982717235,
      "grad_norm": 0.2109375,
      "learning_rate": 7.648186914059142e-07,
      "loss": 0.0738,
      "num_tokens": 184147983.0,
      "reward": 0.611979179084301,
      "reward_std": 0.2557366006076336,
      "rewards/accuracy_reward": 0.1979166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 1061
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1502.5937805175781,
      "epoch": 0.40787325972155547,
      "grad_norm": 0.23828125,
      "learning_rate": 7.642880830534306e-07,
      "loss": 0.1042,
      "num_tokens": 184308504.0,
      "reward": 0.5312500074505806,
      "reward_std": 0.25862899981439114,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333432674408,
      "step": 1062
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.8229675292969,
      "epoch": 0.4082573211713874,
      "grad_norm": 0.23828125,
      "learning_rate": 7.63757089112584e-07,
      "loss": 0.0586,
      "num_tokens": 184468551.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.1587177086621523,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833432674408,
      "step": 1063
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.9375305175781,
      "epoch": 0.4086413826212194,
      "grad_norm": 0.1787109375,
      "learning_rate": 7.632257105388408e-07,
      "loss": 0.0484,
      "num_tokens": 184638329.0,
      "reward": 0.526041679084301,
      "reward_std": 0.2690668869763613,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1064
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.1146240234375,
      "epoch": 0.40902544407105135,
      "grad_norm": 0.169921875,
      "learning_rate": 7.626939482883595e-07,
      "loss": 0.0715,
      "num_tokens": 184817900.0,
      "reward": 0.486979179084301,
      "reward_std": 0.16244950890541077,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1065
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.1042175292969,
      "epoch": 0.40940950552088334,
      "grad_norm": 0.2021484375,
      "learning_rate": 7.621618033179892e-07,
      "loss": 0.0746,
      "num_tokens": 184997222.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.20769778918474913,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 1066
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.8750610351562,
      "epoch": 0.40979356697071534,
      "grad_norm": 0.19140625,
      "learning_rate": 7.616292765852675e-07,
      "loss": 0.078,
      "num_tokens": 185177106.0,
      "reward": 0.432291679084301,
      "reward_std": 0.15465350076556206,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 1067
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.1875610351562,
      "epoch": 0.4101776284205473,
      "grad_norm": 0.224609375,
      "learning_rate": 7.610963690484192e-07,
      "loss": 0.1049,
      "num_tokens": 185351396.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.18495119363069534,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1068
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1506.9792175292969,
      "epoch": 0.4105616898703793,
      "grad_norm": 0.23046875,
      "learning_rate": 7.605630816663537e-07,
      "loss": 0.0904,
      "num_tokens": 185509898.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.1753578893840313,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 1069
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.6146240234375,
      "epoch": 0.4109457513202112,
      "grad_norm": 0.2255859375,
      "learning_rate": 7.600294153986647e-07,
      "loss": 0.161,
      "num_tokens": 185678117.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.25007928907871246,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 1070
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1485.4896545410156,
      "epoch": 0.4113298127700432,
      "grad_norm": 0.2060546875,
      "learning_rate": 7.594953712056271e-07,
      "loss": 0.1045,
      "num_tokens": 185834180.0,
      "reward": 0.5677083432674408,
      "reward_std": 0.26119064912199974,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250074505806,
      "step": 1071
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.8958740234375,
      "epoch": 0.41171387421987515,
      "grad_norm": 0.1826171875,
      "learning_rate": 7.589609500481959e-07,
      "loss": 0.1093,
      "num_tokens": 186009002.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.16801732778549194,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1072
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.0417175292969,
      "epoch": 0.41209793566970715,
      "grad_norm": 0.212890625,
      "learning_rate": 7.584261528880047e-07,
      "loss": 0.1255,
      "num_tokens": 186193590.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.14937817864120007,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 1073
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.6458740234375,
      "epoch": 0.41248199711953915,
      "grad_norm": 0.2412109375,
      "learning_rate": 7.578909806873634e-07,
      "loss": 0.0845,
      "num_tokens": 186365948.0,
      "reward": 0.5208333507180214,
      "reward_std": 0.22780223749578,
      "rewards/accuracy_reward": 0.1249999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1074
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.9271545410156,
      "epoch": 0.4128660585693711,
      "grad_norm": 0.2373046875,
      "learning_rate": 7.573554344092568e-07,
      "loss": 0.0697,
      "num_tokens": 186527885.0,
      "reward": 0.4661458358168602,
      "reward_std": 0.17321879044175148,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1075
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.8854675292969,
      "epoch": 0.4132501200192031,
      "grad_norm": 0.2001953125,
      "learning_rate": 7.56819515017343e-07,
      "loss": 0.121,
      "num_tokens": 186698250.0,
      "reward": 0.604166679084301,
      "reward_std": 0.2881784290075302,
      "rewards/accuracy_reward": 0.17708333022892475,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1076
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.3750305175781,
      "epoch": 0.413634181469035,
      "grad_norm": 0.20703125,
      "learning_rate": 7.56283223475951e-07,
      "loss": 0.0708,
      "num_tokens": 186861006.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.16325468104332685,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541567325592,
      "step": 1077
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.6458740234375,
      "epoch": 0.414018242918867,
      "grad_norm": 0.2236328125,
      "learning_rate": 7.557465607500799e-07,
      "loss": 0.1199,
      "num_tokens": 187045108.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.22545762546360493,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 1078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.2604370117188,
      "epoch": 0.414402304368699,
      "grad_norm": 0.2333984375,
      "learning_rate": 7.552095278053968e-07,
      "loss": 0.0852,
      "num_tokens": 187201069.0,
      "reward": 0.5651041716337204,
      "reward_std": 0.24226953648030758,
      "rewards/accuracy_reward": 0.1562500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1079
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.4792175292969,
      "epoch": 0.41478636581853096,
      "grad_norm": 0.212890625,
      "learning_rate": 7.546721256082343e-07,
      "loss": 0.0952,
      "num_tokens": 187368307.0,
      "reward": 0.505208358168602,
      "reward_std": 0.16165141575038433,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1080
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.0000305175781,
      "epoch": 0.41517042726836295,
      "grad_norm": 0.2353515625,
      "learning_rate": 7.541343551255903e-07,
      "loss": 0.124,
      "num_tokens": 187532251.0,
      "reward": 0.5416666716337204,
      "reward_std": 0.23917845636606216,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 1081
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1448.5417175292969,
      "epoch": 0.4155544887181949,
      "grad_norm": 0.2431640625,
      "learning_rate": 7.535962173251246e-07,
      "loss": 0.0535,
      "num_tokens": 187685839.0,
      "reward": 0.658854179084301,
      "reward_std": 0.2524518668651581,
      "rewards/accuracy_reward": 0.2604166604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1082
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1538.5000610351562,
      "epoch": 0.4159385501680269,
      "grad_norm": 0.2138671875,
      "learning_rate": 7.530577131751586e-07,
      "loss": 0.0796,
      "num_tokens": 187846735.0,
      "reward": 0.497395858168602,
      "reward_std": 0.19678442925214767,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1083
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.4687805175781,
      "epoch": 0.4163226116178589,
      "grad_norm": 0.23046875,
      "learning_rate": 7.525188436446723e-07,
      "loss": 0.1406,
      "num_tokens": 188027148.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.16785479709506035,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 1084
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.4792175292969,
      "epoch": 0.4167066730676908,
      "grad_norm": 0.1875,
      "learning_rate": 7.51979609703304e-07,
      "loss": 0.0812,
      "num_tokens": 188205746.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.2689470946788788,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1085
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.9479675292969,
      "epoch": 0.4170907345175228,
      "grad_norm": 0.2060546875,
      "learning_rate": 7.514400123213467e-07,
      "loss": 0.0628,
      "num_tokens": 188383581.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.2563222646713257,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1086
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.1667175292969,
      "epoch": 0.41747479596735476,
      "grad_norm": 0.1962890625,
      "learning_rate": 7.50900052469748e-07,
      "loss": 0.0235,
      "num_tokens": 188547197.0,
      "reward": 0.5703125223517418,
      "reward_std": 0.12794783525168896,
      "rewards/accuracy_reward": 0.1458333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 1087
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.7292175292969,
      "epoch": 0.41785885741718676,
      "grad_norm": 0.205078125,
      "learning_rate": 7.503597311201076e-07,
      "loss": 0.1039,
      "num_tokens": 188716227.0,
      "reward": 0.552083358168602,
      "reward_std": 0.21515101194381714,
      "rewards/accuracy_reward": 0.14583333022892475,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1088
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.5312805175781,
      "epoch": 0.4182429188670187,
      "grad_norm": 0.2197265625,
      "learning_rate": 7.498190492446758e-07,
      "loss": 0.101,
      "num_tokens": 188890550.0,
      "reward": 0.432291679084301,
      "reward_std": 0.18951749429106712,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1089
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.3021240234375,
      "epoch": 0.4186269803168507,
      "grad_norm": 0.1826171875,
      "learning_rate": 7.492780078163513e-07,
      "loss": 0.0707,
      "num_tokens": 189072403.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.24412602558732033,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1090
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.7083587646484,
      "epoch": 0.4190110417666827,
      "grad_norm": 0.234375,
      "learning_rate": 7.4873660780868e-07,
      "loss": 0.0755,
      "num_tokens": 189234799.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.160383690148592,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1091
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1822.5104675292969,
      "epoch": 0.41939510321651463,
      "grad_norm": 0.1923828125,
      "learning_rate": 7.481948501958531e-07,
      "loss": 0.1088,
      "num_tokens": 189423544.0,
      "reward": 0.3671875074505806,
      "reward_std": 0.11780421063303947,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 1092
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.7396240234375,
      "epoch": 0.4197791646663466,
      "grad_norm": 0.20703125,
      "learning_rate": 7.476527359527051e-07,
      "loss": 0.109,
      "num_tokens": 189599279.0,
      "reward": 0.4765625223517418,
      "reward_std": 0.21401439979672432,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 1093
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.9583435058594,
      "epoch": 0.42016322611617857,
      "grad_norm": 0.181640625,
      "learning_rate": 7.471102660547124e-07,
      "loss": 0.1076,
      "num_tokens": 189774115.0,
      "reward": 0.4036458507180214,
      "reward_std": 0.15821388689801097,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1094
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.9688110351562,
      "epoch": 0.42054728756601056,
      "grad_norm": 0.2275390625,
      "learning_rate": 7.465674414779909e-07,
      "loss": 0.1308,
      "num_tokens": 189944976.0,
      "reward": 0.4453125223517418,
      "reward_std": 0.19444776885211468,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1095
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1673.7396240234375,
      "epoch": 0.42093134901584256,
      "grad_norm": 0.205078125,
      "learning_rate": 7.460242631992956e-07,
      "loss": 0.1124,
      "num_tokens": 190120231.0,
      "reward": 0.5911458507180214,
      "reward_std": 0.2881063390523195,
      "rewards/accuracy_reward": 0.1770833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1096
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.0000305175781,
      "epoch": 0.4213154104656745,
      "grad_norm": 0.2236328125,
      "learning_rate": 7.45480732196017e-07,
      "loss": 0.1125,
      "num_tokens": 190295287.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.17323173582553864,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 1097
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.7188110351562,
      "epoch": 0.4216994719155065,
      "grad_norm": 0.2177734375,
      "learning_rate": 7.449368494461807e-07,
      "loss": 0.1111,
      "num_tokens": 190462948.0,
      "reward": 0.5598958507180214,
      "reward_std": 0.3108360879123211,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1098
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1357.6458435058594,
      "epoch": 0.42208353336533844,
      "grad_norm": 0.240234375,
      "learning_rate": 7.443926159284453e-07,
      "loss": 0.142,
      "num_tokens": 190607722.0,
      "reward": 0.6093750298023224,
      "reward_std": 0.29086878150701523,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5052083283662796,
      "step": 1099
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.1146240234375,
      "epoch": 0.42246759481517043,
      "grad_norm": 0.23046875,
      "learning_rate": 7.438480326221007e-07,
      "loss": 0.0999,
      "num_tokens": 190764533.0,
      "reward": 0.5468750223517418,
      "reward_std": 0.2124207727611065,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.484375,
      "step": 1100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1814.3438110351562,
      "epoch": 0.4228516562650024,
      "grad_norm": 0.2060546875,
      "learning_rate": 7.433031005070658e-07,
      "loss": 0.0917,
      "num_tokens": 190954862.0,
      "reward": 0.3671875074505806,
      "reward_std": 0.14845656417310238,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375,
      "step": 1101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.1354675292969,
      "epoch": 0.42323571771483437,
      "grad_norm": 0.234375,
      "learning_rate": 7.427578205638874e-07,
      "loss": 0.1147,
      "num_tokens": 191121467.0,
      "reward": 0.4973958358168602,
      "reward_std": 0.24456734396517277,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.6146240234375,
      "epoch": 0.42361977916466637,
      "grad_norm": 0.25,
      "learning_rate": 7.42212193773738e-07,
      "loss": 0.0932,
      "num_tokens": 191286214.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.1445080148987472,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.3541870117188,
      "epoch": 0.4240038406144983,
      "grad_norm": 0.24609375,
      "learning_rate": 7.416662211184146e-07,
      "loss": 0.1376,
      "num_tokens": 191460872.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.22214143723249435,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 1104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.90625,
      "epoch": 0.4243879020643303,
      "grad_norm": 0.1689453125,
      "learning_rate": 7.411199035803359e-07,
      "loss": 0.0622,
      "num_tokens": 191614127.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.108740727417171,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 1105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1806.7709045410156,
      "epoch": 0.42477196351416224,
      "grad_norm": 0.1787109375,
      "learning_rate": 7.40573242142542e-07,
      "loss": 0.0815,
      "num_tokens": 191802049.0,
      "reward": 0.3828125074505806,
      "reward_std": 0.1423029894940555,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.1354675292969,
      "epoch": 0.42515602496399424,
      "grad_norm": 0.25,
      "learning_rate": 7.40026237788691e-07,
      "loss": 0.1717,
      "num_tokens": 191961246.0,
      "reward": 0.5729166865348816,
      "reward_std": 0.27692143991589546,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4791666641831398,
      "step": 1107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.0729370117188,
      "epoch": 0.42554008641382624,
      "grad_norm": 0.2177734375,
      "learning_rate": 7.394788915030586e-07,
      "loss": 0.1244,
      "num_tokens": 192126773.0,
      "reward": 0.528645858168602,
      "reward_std": 0.29343593679368496,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 1108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.1875305175781,
      "epoch": 0.4259241478636582,
      "grad_norm": 0.2314453125,
      "learning_rate": 7.389312042705356e-07,
      "loss": 0.1119,
      "num_tokens": 192284367.0,
      "reward": 0.4895833507180214,
      "reward_std": 0.17403815686702728,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 1109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.6666870117188,
      "epoch": 0.42630820931349017,
      "grad_norm": 0.1708984375,
      "learning_rate": 7.383831770766263e-07,
      "loss": 0.0966,
      "num_tokens": 192466503.0,
      "reward": 0.4557291716337204,
      "reward_std": 0.12407433614134789,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.9166870117188,
      "epoch": 0.4266922707633221,
      "grad_norm": 0.2275390625,
      "learning_rate": 7.378348109074467e-07,
      "loss": 0.125,
      "num_tokens": 192648839.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.2465786151587963,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.6354675292969,
      "epoch": 0.4270763322131541,
      "grad_norm": 0.19140625,
      "learning_rate": 7.372861067497224e-07,
      "loss": 0.0581,
      "num_tokens": 192824500.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.17927787080407143,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.4166870117188,
      "epoch": 0.4274603936629861,
      "grad_norm": 0.2099609375,
      "learning_rate": 7.367370655907881e-07,
      "loss": 0.0502,
      "num_tokens": 192997132.0,
      "reward": 0.5390625298023224,
      "reward_std": 0.2778191305696964,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1502.6354370117188,
      "epoch": 0.42784445511281805,
      "grad_norm": 0.2314453125,
      "learning_rate": 7.361876884185838e-07,
      "loss": 0.1019,
      "num_tokens": 193155161.0,
      "reward": 0.4791666716337204,
      "reward_std": 0.19055498391389847,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 1114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.6354675292969,
      "epoch": 0.42822851656265004,
      "grad_norm": 0.28515625,
      "learning_rate": 7.356379762216553e-07,
      "loss": 0.069,
      "num_tokens": 193320774.0,
      "reward": 0.5494791939854622,
      "reward_std": 0.20855917409062386,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 1115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.4896240234375,
      "epoch": 0.428612578012482,
      "grad_norm": 0.1943359375,
      "learning_rate": 7.350879299891498e-07,
      "loss": 0.0607,
      "num_tokens": 193508261.0,
      "reward": 0.4895833432674408,
      "reward_std": 0.25022698007524014,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 1116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1419.0729370117188,
      "epoch": 0.428996639462314,
      "grad_norm": 0.2470703125,
      "learning_rate": 7.34537550710817e-07,
      "loss": 0.0575,
      "num_tokens": 193658180.0,
      "reward": 0.8125000149011612,
      "reward_std": 0.2692483477294445,
      "rewards/accuracy_reward": 0.34375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687500074505806,
      "step": 1117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.9375610351562,
      "epoch": 0.4293807009121459,
      "grad_norm": 0.21484375,
      "learning_rate": 7.339868393770049e-07,
      "loss": 0.1294,
      "num_tokens": 193834798.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.199541836977005,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1376.2396240234375,
      "epoch": 0.4297647623619779,
      "grad_norm": 0.25390625,
      "learning_rate": 7.334357969786594e-07,
      "loss": 0.0677,
      "num_tokens": 193980725.0,
      "reward": 0.5182291939854622,
      "reward_std": 0.18751344084739685,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458283662796,
      "step": 1119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.4896240234375,
      "epoch": 0.4301488238118099,
      "grad_norm": 0.205078125,
      "learning_rate": 7.328844245073221e-07,
      "loss": 0.1168,
      "num_tokens": 194162860.0,
      "reward": 0.3645833432674408,
      "reward_std": 0.14055148046463728,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666716337204,
      "step": 1120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.2396240234375,
      "epoch": 0.43053288526164185,
      "grad_norm": 0.18359375,
      "learning_rate": 7.323327229551284e-07,
      "loss": 0.1111,
      "num_tokens": 194348955.0,
      "reward": 0.369791679084301,
      "reward_std": 0.15627874061465263,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 1121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1476.9583740234375,
      "epoch": 0.43091694671147385,
      "grad_norm": 0.2080078125,
      "learning_rate": 7.317806933148057e-07,
      "loss": 0.0986,
      "num_tokens": 194503495.0,
      "reward": 0.528645858168602,
      "reward_std": 0.25709258019924164,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 1122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1535.7396240234375,
      "epoch": 0.4313010081613058,
      "grad_norm": 0.255859375,
      "learning_rate": 7.312283365796726e-07,
      "loss": 0.0617,
      "num_tokens": 194664350.0,
      "reward": 0.5937500149011612,
      "reward_std": 0.28925383649766445,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4895833283662796,
      "step": 1123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.0313110351562,
      "epoch": 0.4316850696111378,
      "grad_norm": 0.19140625,
      "learning_rate": 7.306756537436349e-07,
      "loss": 0.1427,
      "num_tokens": 194846985.0,
      "reward": 0.4661458358168602,
      "reward_std": 0.265540175139904,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.5625305175781,
      "epoch": 0.4320691310609698,
      "grad_norm": 0.169921875,
      "learning_rate": 7.301226458011866e-07,
      "loss": 0.0964,
      "num_tokens": 195032719.0,
      "reward": 0.455729179084301,
      "reward_std": 0.2855034349486232,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.9791870117188,
      "epoch": 0.4324531925108017,
      "grad_norm": 0.216796875,
      "learning_rate": 7.295693137474056e-07,
      "loss": 0.1273,
      "num_tokens": 195203405.0,
      "reward": 0.5937500149011612,
      "reward_std": 0.3338296711444855,
      "rewards/accuracy_reward": 0.1770833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.8229675292969,
      "epoch": 0.4328372539606337,
      "grad_norm": 0.1875,
      "learning_rate": 7.290156585779535e-07,
      "loss": 0.0857,
      "num_tokens": 195370444.0,
      "reward": 0.5130208358168602,
      "reward_std": 0.25868134573102,
      "rewards/accuracy_reward": 0.10416666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.6979370117188,
      "epoch": 0.43322131541046566,
      "grad_norm": 0.265625,
      "learning_rate": 7.284616812890734e-07,
      "loss": 0.0569,
      "num_tokens": 195536295.0,
      "reward": 0.4375000223517418,
      "reward_std": 0.13581100339069963,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 1128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1799.7917175292969,
      "epoch": 0.43360537686029765,
      "grad_norm": 0.1796875,
      "learning_rate": 7.279073828775877e-07,
      "loss": 0.082,
      "num_tokens": 195726051.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.16302297078073025,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916567325592,
      "step": 1129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1492.6250610351562,
      "epoch": 0.4339894383101296,
      "grad_norm": 0.2470703125,
      "learning_rate": 7.273527643408969e-07,
      "loss": 0.1217,
      "num_tokens": 195882839.0,
      "reward": 0.479166679084301,
      "reward_std": 0.18450861424207687,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.8854675292969,
      "epoch": 0.4343734997599616,
      "grad_norm": 0.248046875,
      "learning_rate": 7.267978266769772e-07,
      "loss": 0.0919,
      "num_tokens": 196049852.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.31070946902036667,
      "rewards/accuracy_reward": 0.14583333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 1131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.5208740234375,
      "epoch": 0.4347575612097936,
      "grad_norm": 0.232421875,
      "learning_rate": 7.262425708843797e-07,
      "loss": 0.1096,
      "num_tokens": 196213614.0,
      "reward": 0.5078125223517418,
      "reward_std": 0.23233642801642418,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 1132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.1562805175781,
      "epoch": 0.4351416226596255,
      "grad_norm": 0.2109375,
      "learning_rate": 7.25686997962227e-07,
      "loss": 0.1383,
      "num_tokens": 196387117.0,
      "reward": 0.5520833507180214,
      "reward_std": 0.2080192193388939,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.9375305175781,
      "epoch": 0.4355256841094575,
      "grad_norm": 0.224609375,
      "learning_rate": 7.251311089102132e-07,
      "loss": 0.0922,
      "num_tokens": 196545167.0,
      "reward": 0.6979166865348816,
      "reward_std": 0.32306353747844696,
      "rewards/accuracy_reward": 0.265625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 1134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.2083740234375,
      "epoch": 0.43590974555928946,
      "grad_norm": 0.2109375,
      "learning_rate": 7.245749047286006e-07,
      "loss": 0.1046,
      "num_tokens": 196731419.0,
      "reward": 0.424479179084301,
      "reward_std": 0.16428370960056782,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.1875305175781,
      "epoch": 0.43629380700912146,
      "grad_norm": 0.21875,
      "learning_rate": 7.240183864182189e-07,
      "loss": 0.0933,
      "num_tokens": 196895717.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.19410105422139168,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1886.8437805175781,
      "epoch": 0.43667786845895346,
      "grad_norm": 0.154296875,
      "learning_rate": 7.23461554980463e-07,
      "loss": 0.0568,
      "num_tokens": 197096198.0,
      "reward": 0.3255208432674408,
      "reward_std": 0.09408319555222988,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3151041641831398,
      "step": 1137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1656.5312805175781,
      "epoch": 0.4370619299087854,
      "grad_norm": 0.1982421875,
      "learning_rate": 7.229044114172914e-07,
      "loss": 0.082,
      "num_tokens": 197270769.0,
      "reward": 0.5598958432674408,
      "reward_std": 0.29461587592959404,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.3854370117188,
      "epoch": 0.4374459913586174,
      "grad_norm": 0.1767578125,
      "learning_rate": 7.223469567312237e-07,
      "loss": 0.0902,
      "num_tokens": 197456334.0,
      "reward": 0.3776041716337204,
      "reward_std": 0.12892378680408,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875074505806,
      "step": 1139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.9375305175781,
      "epoch": 0.43783005280844933,
      "grad_norm": 0.1904296875,
      "learning_rate": 7.217891919253399e-07,
      "loss": 0.1053,
      "num_tokens": 197641208.0,
      "reward": 0.3697916865348816,
      "reward_std": 0.10773391090333462,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916567325592,
      "step": 1140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.3854370117188,
      "epoch": 0.43821411425828133,
      "grad_norm": 0.20703125,
      "learning_rate": 7.212311180032774e-07,
      "loss": 0.1142,
      "num_tokens": 197806597.0,
      "reward": 0.5781250223517418,
      "reward_std": 0.24841829389333725,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 1141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1435.5312805175781,
      "epoch": 0.4385981757081133,
      "grad_norm": 0.234375,
      "learning_rate": 7.206727359692307e-07,
      "loss": 0.0786,
      "num_tokens": 197958184.0,
      "reward": 0.6041666865348816,
      "reward_std": 0.27618785202503204,
      "rewards/accuracy_reward": 0.14583333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 1142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.1146240234375,
      "epoch": 0.43898223715794527,
      "grad_norm": 0.263671875,
      "learning_rate": 7.201140468279479e-07,
      "loss": 0.0829,
      "num_tokens": 198116067.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.1939704455435276,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.5312805175781,
      "epoch": 0.43936629860777726,
      "grad_norm": 0.21484375,
      "learning_rate": 7.195550515847302e-07,
      "loss": 0.0709,
      "num_tokens": 198285870.0,
      "reward": 0.401041679084301,
      "reward_std": 0.11075925827026367,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.9896545410156,
      "epoch": 0.4397503600576092,
      "grad_norm": 0.2255859375,
      "learning_rate": 7.189957512454294e-07,
      "loss": 0.09,
      "num_tokens": 198452245.0,
      "reward": 0.536458358168602,
      "reward_std": 0.15857519675046206,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1572.9375305175781,
      "epoch": 0.4401344215074412,
      "grad_norm": 0.25390625,
      "learning_rate": 7.184361468164463e-07,
      "loss": 0.0563,
      "num_tokens": 198616615.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.2554282732307911,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666567325592,
      "step": 1146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.4583740234375,
      "epoch": 0.44051848295727314,
      "grad_norm": 0.2099609375,
      "learning_rate": 7.17876239304729e-07,
      "loss": 0.0842,
      "num_tokens": 198782947.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.20691360346972942,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 1147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.7292175292969,
      "epoch": 0.44090254440710513,
      "grad_norm": 0.23046875,
      "learning_rate": 7.173160297177708e-07,
      "loss": 0.1027,
      "num_tokens": 198953769.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.2274113316088915,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541567325592,
      "step": 1148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.2084045410156,
      "epoch": 0.44128660585693713,
      "grad_norm": 0.1953125,
      "learning_rate": 7.167555190636084e-07,
      "loss": 0.0468,
      "num_tokens": 199120541.0,
      "reward": 0.518229179084301,
      "reward_std": 0.2437471430748701,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 1149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.7604675292969,
      "epoch": 0.44167066730676907,
      "grad_norm": 0.2236328125,
      "learning_rate": 7.161947083508206e-07,
      "loss": 0.0997,
      "num_tokens": 199299534.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.22948714718222618,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 1150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.9375,
      "epoch": 0.44205472875660107,
      "grad_norm": 0.23828125,
      "learning_rate": 7.156335985885261e-07,
      "loss": 0.1247,
      "num_tokens": 199459152.0,
      "reward": 0.5494791716337204,
      "reward_std": 0.1664994079619646,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 1151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1489.6041870117188,
      "epoch": 0.442438790206433,
      "grad_norm": 0.2353515625,
      "learning_rate": 7.150721907863816e-07,
      "loss": 0.194,
      "num_tokens": 199617074.0,
      "reward": 0.5598958432674408,
      "reward_std": 0.27690746635198593,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 1152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.8229675292969,
      "epoch": 0.442822851656265,
      "grad_norm": 0.21484375,
      "learning_rate": 7.145104859545798e-07,
      "loss": 0.0868,
      "num_tokens": 199798785.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.16811983101069927,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 1153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.2396240234375,
      "epoch": 0.443206913106097,
      "grad_norm": 0.2021484375,
      "learning_rate": 7.139484851038483e-07,
      "loss": 0.0764,
      "num_tokens": 199967984.0,
      "reward": 0.6796875149011612,
      "reward_std": 0.26232248544692993,
      "rewards/accuracy_reward": 0.2708333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.1146545410156,
      "epoch": 0.44359097455592894,
      "grad_norm": 0.2099609375,
      "learning_rate": 7.133861892454473e-07,
      "loss": 0.1361,
      "num_tokens": 200135747.0,
      "reward": 0.533854179084301,
      "reward_std": 0.25310590118169785,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1863.3958740234375,
      "epoch": 0.44397503600576094,
      "grad_norm": 0.166015625,
      "learning_rate": 7.128235993911675e-07,
      "loss": 0.0616,
      "num_tokens": 200326617.0,
      "reward": 0.401041679084301,
      "reward_std": 0.18277988024055958,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 1156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.0104370117188,
      "epoch": 0.4443590974555929,
      "grad_norm": 0.251953125,
      "learning_rate": 7.12260716553329e-07,
      "loss": 0.1087,
      "num_tokens": 200485722.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.20282099954783916,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 1157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.1041870117188,
      "epoch": 0.4447431589054249,
      "grad_norm": 0.234375,
      "learning_rate": 7.116975417447788e-07,
      "loss": 0.1065,
      "num_tokens": 200658196.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.188152976334095,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.7916870117188,
      "epoch": 0.4451272203552568,
      "grad_norm": 0.26953125,
      "learning_rate": 7.111340759788897e-07,
      "loss": 0.0981,
      "num_tokens": 200828728.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.21780724450945854,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.6771240234375,
      "epoch": 0.4455112818050888,
      "grad_norm": 0.232421875,
      "learning_rate": 7.105703202695577e-07,
      "loss": 0.1162,
      "num_tokens": 200999345.0,
      "reward": 0.5937500223517418,
      "reward_std": 0.29534125328063965,
      "rewards/accuracy_reward": 0.17708333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1250.7708740234375,
      "epoch": 0.4458953432549208,
      "grad_norm": 0.228515625,
      "learning_rate": 7.100062756312001e-07,
      "loss": 0.0701,
      "num_tokens": 201133891.0,
      "reward": 0.7760416865348816,
      "reward_std": 0.4160642623901367,
      "rewards/accuracy_reward": 0.2708333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5052083358168602,
      "step": 1161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.6042175292969,
      "epoch": 0.44627940470475275,
      "grad_norm": 0.2333984375,
      "learning_rate": 7.094419430787553e-07,
      "loss": 0.1358,
      "num_tokens": 201301597.0,
      "reward": 0.588541679084301,
      "reward_std": 0.22758353874087334,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 1162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1413.7812805175781,
      "epoch": 0.44666346615458474,
      "grad_norm": 0.259765625,
      "learning_rate": 7.088773236276785e-07,
      "loss": 0.1621,
      "num_tokens": 201451512.0,
      "reward": 0.5885416939854622,
      "reward_std": 0.22499222960323095,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4947916716337204,
      "step": 1163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.5833740234375,
      "epoch": 0.4470475276044167,
      "grad_norm": 0.255859375,
      "learning_rate": 7.083124182939419e-07,
      "loss": 0.1498,
      "num_tokens": 201611344.0,
      "reward": 0.4583333358168602,
      "reward_std": 0.18472165241837502,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.1250305175781,
      "epoch": 0.4474315890542487,
      "grad_norm": 0.2490234375,
      "learning_rate": 7.077472280940319e-07,
      "loss": 0.1426,
      "num_tokens": 201782140.0,
      "reward": 0.5390625074505806,
      "reward_std": 0.3479653038084507,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 1165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.1458740234375,
      "epoch": 0.4478156505040807,
      "grad_norm": 0.302734375,
      "learning_rate": 7.071817540449473e-07,
      "loss": 0.0991,
      "num_tokens": 201934562.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.25302909314632416,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458432674408,
      "step": 1166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.0937805175781,
      "epoch": 0.4481997119539126,
      "grad_norm": 0.193359375,
      "learning_rate": 7.066159971641981e-07,
      "loss": 0.1251,
      "num_tokens": 202118731.0,
      "reward": 0.4036458358168602,
      "reward_std": 0.17521815188229084,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1683.2708435058594,
      "epoch": 0.4485837734037446,
      "grad_norm": 0.2314453125,
      "learning_rate": 7.060499584698032e-07,
      "loss": 0.1081,
      "num_tokens": 202294461.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.17696005664765835,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.8125305175781,
      "epoch": 0.44896783485357655,
      "grad_norm": 0.259765625,
      "learning_rate": 7.054836389802876e-07,
      "loss": 0.0738,
      "num_tokens": 202458467.0,
      "reward": 0.440104179084301,
      "reward_std": 0.20197847671806812,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1424.187515258789,
      "epoch": 0.44935189630340855,
      "grad_norm": 0.240234375,
      "learning_rate": 7.049170397146831e-07,
      "loss": 0.0557,
      "num_tokens": 202609173.0,
      "reward": 0.5911458432674408,
      "reward_std": 0.25928136333823204,
      "rewards/accuracy_reward": 0.10416666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4869791641831398,
      "step": 1170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.8437805175781,
      "epoch": 0.44973595775324055,
      "grad_norm": 0.154296875,
      "learning_rate": 7.043501616925237e-07,
      "loss": 0.0875,
      "num_tokens": 202793286.0,
      "reward": 0.3958333507180214,
      "reward_std": 0.17110459879040718,
      "rewards/accuracy_reward": 0.046875000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 1171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.1458435058594,
      "epoch": 0.4501200192030725,
      "grad_norm": 0.201171875,
      "learning_rate": 7.037830059338457e-07,
      "loss": 0.1306,
      "num_tokens": 202967580.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.26839040219783783,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.0520935058594,
      "epoch": 0.4505040806529045,
      "grad_norm": 0.234375,
      "learning_rate": 7.032155734591845e-07,
      "loss": 0.0717,
      "num_tokens": 203143689.0,
      "reward": 0.4791666716337204,
      "reward_std": 0.2628548927605152,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.9167175292969,
      "epoch": 0.4508881421027364,
      "grad_norm": 0.1923828125,
      "learning_rate": 7.026478652895742e-07,
      "loss": 0.1015,
      "num_tokens": 203302745.0,
      "reward": 0.5572916939854622,
      "reward_std": 0.2457890771329403,
      "rewards/accuracy_reward": 0.12500000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 1174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1840.9271240234375,
      "epoch": 0.4512722035525684,
      "grad_norm": 0.1748046875,
      "learning_rate": 7.020798824465444e-07,
      "loss": 0.1047,
      "num_tokens": 203495170.0,
      "reward": 0.3776041716337204,
      "reward_std": 0.14563091192394495,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.1459045410156,
      "epoch": 0.45165626500240036,
      "grad_norm": 0.2421875,
      "learning_rate": 7.015116259521195e-07,
      "loss": 0.114,
      "num_tokens": 203664264.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.2885156311094761,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.7604370117188,
      "epoch": 0.45204032645223235,
      "grad_norm": 0.22265625,
      "learning_rate": 7.009430968288155e-07,
      "loss": 0.115,
      "num_tokens": 203836913.0,
      "reward": 0.4401041865348816,
      "reward_std": 0.1694733388721943,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 1177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1485.3333435058594,
      "epoch": 0.45242438790206435,
      "grad_norm": 0.28125,
      "learning_rate": 7.003742960996398e-07,
      "loss": 0.11,
      "num_tokens": 203992665.0,
      "reward": 0.6640625074505806,
      "reward_std": 0.24858376011252403,
      "rewards/accuracy_reward": 0.22916666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1410.9688110351562,
      "epoch": 0.4528084493518963,
      "grad_norm": 0.2578125,
      "learning_rate": 6.998052247880878e-07,
      "loss": 0.1765,
      "num_tokens": 204143438.0,
      "reward": 0.6744791716337204,
      "reward_std": 0.3209725357592106,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4869791641831398,
      "step": 1179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.5521240234375,
      "epoch": 0.4531925108017283,
      "grad_norm": 0.2060546875,
      "learning_rate": 6.992358839181425e-07,
      "loss": 0.0504,
      "num_tokens": 204320371.0,
      "reward": 0.4895833358168602,
      "reward_std": 0.17472468595951796,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.3125305175781,
      "epoch": 0.45357657225156023,
      "grad_norm": 0.1884765625,
      "learning_rate": 6.986662745142714e-07,
      "loss": 0.1095,
      "num_tokens": 204498145.0,
      "reward": 0.4375000223517418,
      "reward_std": 0.18794495798647404,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.1042175292969,
      "epoch": 0.4539606337013922,
      "grad_norm": 0.220703125,
      "learning_rate": 6.980963976014251e-07,
      "loss": 0.092,
      "num_tokens": 204672267.0,
      "reward": 0.497395858168602,
      "reward_std": 0.23133927211165428,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125074505806,
      "step": 1182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.5000610351562,
      "epoch": 0.4543446951512242,
      "grad_norm": 0.23828125,
      "learning_rate": 6.975262542050363e-07,
      "loss": 0.0687,
      "num_tokens": 204825803.0,
      "reward": 0.5520833507180214,
      "reward_std": 0.21765738725662231,
      "rewards/accuracy_reward": 0.1249999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.4063110351562,
      "epoch": 0.45472875660105616,
      "grad_norm": 0.2255859375,
      "learning_rate": 6.969558453510165e-07,
      "loss": 0.1316,
      "num_tokens": 204995530.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.21975987404584885,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1481.9896240234375,
      "epoch": 0.45511281805088816,
      "grad_norm": 0.255859375,
      "learning_rate": 6.963851720657551e-07,
      "loss": 0.1212,
      "num_tokens": 205150897.0,
      "reward": 0.5390625074505806,
      "reward_std": 0.23999305069446564,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.0208740234375,
      "epoch": 0.4554968795007201,
      "grad_norm": 0.232421875,
      "learning_rate": 6.958142353761173e-07,
      "loss": 0.1076,
      "num_tokens": 205329019.0,
      "reward": 0.4401041716337204,
      "reward_std": 0.21375204995274544,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 1186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1554.5417175292969,
      "epoch": 0.4558809409505521,
      "grad_norm": 0.2021484375,
      "learning_rate": 6.952430363094423e-07,
      "loss": 0.1077,
      "num_tokens": 205493239.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.1855798326432705,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 1187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.7084045410156,
      "epoch": 0.45626500240038403,
      "grad_norm": 0.23046875,
      "learning_rate": 6.946715758935413e-07,
      "loss": 0.1614,
      "num_tokens": 205663795.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.20019008591771126,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.7500305175781,
      "epoch": 0.45664906385021603,
      "grad_norm": 0.2138671875,
      "learning_rate": 6.940998551566962e-07,
      "loss": 0.0716,
      "num_tokens": 205819699.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.1264797654002905,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 1189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.9479675292969,
      "epoch": 0.457033125300048,
      "grad_norm": 0.1943359375,
      "learning_rate": 6.935278751276567e-07,
      "loss": 0.0814,
      "num_tokens": 205996758.0,
      "reward": 0.463541679084301,
      "reward_std": 0.1657528430223465,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.8229675292969,
      "epoch": 0.45741718674987997,
      "grad_norm": 0.2119140625,
      "learning_rate": 6.929556368356394e-07,
      "loss": 0.0799,
      "num_tokens": 206157493.0,
      "reward": 0.5989583432674408,
      "reward_std": 0.2642357498407364,
      "rewards/accuracy_reward": 0.14583333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 1191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.7500305175781,
      "epoch": 0.45780124819971196,
      "grad_norm": 0.208984375,
      "learning_rate": 6.923831413103255e-07,
      "loss": 0.0851,
      "num_tokens": 206324917.0,
      "reward": 0.6328125149011612,
      "reward_std": 0.3631182983517647,
      "rewards/accuracy_reward": 0.1874999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125074505806,
      "step": 1192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.9688110351562,
      "epoch": 0.4581853096495439,
      "grad_norm": 0.1787109375,
      "learning_rate": 6.918103895818594e-07,
      "loss": 0.0687,
      "num_tokens": 206518930.0,
      "reward": 0.4453125,
      "reward_std": 0.18019281327724457,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291716337204,
      "step": 1193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1288.4583740234375,
      "epoch": 0.4585693710993759,
      "grad_norm": 0.279296875,
      "learning_rate": 6.91237382680846e-07,
      "loss": 0.1119,
      "num_tokens": 206656118.0,
      "reward": 0.5755208507180214,
      "reward_std": 0.2504391260445118,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875,
      "step": 1194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.8646240234375,
      "epoch": 0.4589534325492079,
      "grad_norm": 0.189453125,
      "learning_rate": 6.906641216383498e-07,
      "loss": 0.0966,
      "num_tokens": 206825385.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.19864264503121376,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1410.3334045410156,
      "epoch": 0.45933749399903984,
      "grad_norm": 0.25,
      "learning_rate": 6.900906074858921e-07,
      "loss": 0.1137,
      "num_tokens": 206974137.0,
      "reward": 0.6223958432674408,
      "reward_std": 0.23793058097362518,
      "rewards/accuracy_reward": 0.14583333861082792,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765624925494194,
      "step": 1196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1366.6979522705078,
      "epoch": 0.45972155544887183,
      "grad_norm": 0.2578125,
      "learning_rate": 6.895168412554503e-07,
      "loss": 0.0904,
      "num_tokens": 207117828.0,
      "reward": 0.580729179084301,
      "reward_std": 0.23803968355059624,
      "rewards/accuracy_reward": 0.10416666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625149011612,
      "step": 1197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.9896240234375,
      "epoch": 0.4601056168987038,
      "grad_norm": 0.2314453125,
      "learning_rate": 6.889428239794548e-07,
      "loss": 0.1042,
      "num_tokens": 207290627.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.21621563285589218,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.3438110351562,
      "epoch": 0.46048967834853577,
      "grad_norm": 0.21484375,
      "learning_rate": 6.883685566907881e-07,
      "loss": 0.1038,
      "num_tokens": 207472740.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.21903026849031448,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 1199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1445.1250457763672,
      "epoch": 0.46087373979836777,
      "grad_norm": 0.24609375,
      "learning_rate": 6.877940404227822e-07,
      "loss": 0.0759,
      "num_tokens": 207625488.0,
      "reward": 0.6848958432674408,
      "reward_std": 0.28934239968657494,
      "rewards/accuracy_reward": 0.2187499962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 1200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.5625305175781,
      "epoch": 0.4612578012481997,
      "grad_norm": 0.212890625,
      "learning_rate": 6.872192762092177e-07,
      "loss": 0.1122,
      "num_tokens": 207799630.0,
      "reward": 0.5937500223517418,
      "reward_std": 0.2881939299404621,
      "rewards/accuracy_reward": 0.18749999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1667.6354675292969,
      "epoch": 0.4616418626980317,
      "grad_norm": 0.1982421875,
      "learning_rate": 6.866442650843208e-07,
      "loss": 0.0644,
      "num_tokens": 207974915.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.22975797904655337,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.8125610351562,
      "epoch": 0.46202592414786364,
      "grad_norm": 0.1708984375,
      "learning_rate": 6.86069008082762e-07,
      "loss": 0.0536,
      "num_tokens": 208159793.0,
      "reward": 0.3776041716337204,
      "reward_std": 0.12948851753026247,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.4479675292969,
      "epoch": 0.46240998559769564,
      "grad_norm": 0.24609375,
      "learning_rate": 6.854935062396549e-07,
      "loss": 0.1477,
      "num_tokens": 208337884.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.17910443805158138,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1598.6875610351562,
      "epoch": 0.4627940470475276,
      "grad_norm": 0.2119140625,
      "learning_rate": 6.849177605905527e-07,
      "loss": 0.0986,
      "num_tokens": 208504606.0,
      "reward": 0.4609375,
      "reward_std": 0.17694992199540138,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.5625305175781,
      "epoch": 0.4631781084973596,
      "grad_norm": 0.2470703125,
      "learning_rate": 6.843417721714482e-07,
      "loss": 0.1053,
      "num_tokens": 208670828.0,
      "reward": 0.447916679084301,
      "reward_std": 0.17224100790917873,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 1206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.7500305175781,
      "epoch": 0.46356216994719157,
      "grad_norm": 0.197265625,
      "learning_rate": 6.837655420187702e-07,
      "loss": 0.0552,
      "num_tokens": 208839580.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.20075152069330215,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 1207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.6562805175781,
      "epoch": 0.4639462313970235,
      "grad_norm": 0.2451171875,
      "learning_rate": 6.831890711693833e-07,
      "loss": 0.1597,
      "num_tokens": 209001395.0,
      "reward": 0.5885416865348816,
      "reward_std": 0.2951274663209915,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416641831398,
      "step": 1208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.3125610351562,
      "epoch": 0.4643302928468555,
      "grad_norm": 0.2041015625,
      "learning_rate": 6.826123606605845e-07,
      "loss": 0.077,
      "num_tokens": 209161985.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.24402155354619026,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1854.6667175292969,
      "epoch": 0.46471435429668745,
      "grad_norm": 0.173828125,
      "learning_rate": 6.820354115301026e-07,
      "loss": 0.0507,
      "num_tokens": 209354537.0,
      "reward": 0.3697916716337204,
      "reward_std": 0.1347772954031825,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 1210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1244.2812805175781,
      "epoch": 0.46509841574651944,
      "grad_norm": 0.3046875,
      "learning_rate": 6.814582248160952e-07,
      "loss": 0.0687,
      "num_tokens": 209486148.0,
      "reward": 0.5651041865348816,
      "reward_std": 0.23430135846138,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5130208432674408,
      "step": 1211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.7917175292969,
      "epoch": 0.46548247719635144,
      "grad_norm": 0.19921875,
      "learning_rate": 6.80880801557148e-07,
      "loss": 0.0886,
      "num_tokens": 209668256.0,
      "reward": 0.3984375074505806,
      "reward_std": 0.1700836457312107,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 1212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1474.4687805175781,
      "epoch": 0.4658665386461834,
      "grad_norm": 0.2333984375,
      "learning_rate": 6.80303142792272e-07,
      "loss": 0.1567,
      "num_tokens": 209825253.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.25506340339779854,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.3229370117188,
      "epoch": 0.4662506000960154,
      "grad_norm": 0.20703125,
      "learning_rate": 6.797252495609021e-07,
      "loss": 0.105,
      "num_tokens": 210008540.0,
      "reward": 0.4140625,
      "reward_std": 0.24357000179588795,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1446.9896240234375,
      "epoch": 0.4666346615458473,
      "grad_norm": 0.298828125,
      "learning_rate": 6.791471229028948e-07,
      "loss": 0.0783,
      "num_tokens": 210160571.0,
      "reward": 0.5468750149011612,
      "reward_std": 0.2665116898715496,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.0000305175781,
      "epoch": 0.4670187229956793,
      "grad_norm": 0.482421875,
      "learning_rate": 6.785687638585269e-07,
      "loss": 0.0204,
      "num_tokens": 210319819.0,
      "reward": 0.432291679084301,
      "reward_std": 0.13428195752203465,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.432291679084301,
      "step": 1216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.0000610351562,
      "epoch": 0.46740278444551125,
      "grad_norm": 0.220703125,
      "learning_rate": 6.779901734684936e-07,
      "loss": 0.1066,
      "num_tokens": 210494403.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.21967653557658195,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333432674408,
      "step": 1217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.4375305175781,
      "epoch": 0.46778684589534325,
      "grad_norm": 0.23046875,
      "learning_rate": 6.774113527739058e-07,
      "loss": 0.1336,
      "num_tokens": 210656133.0,
      "reward": 0.6015625149011612,
      "reward_std": 0.3401974346488714,
      "rewards/accuracy_reward": 0.1666666716337204,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.2083740234375,
      "epoch": 0.46817090734517525,
      "grad_norm": 0.193359375,
      "learning_rate": 6.768323028162892e-07,
      "loss": 0.0737,
      "num_tokens": 210837577.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.22204073145985603,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 1219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.1979675292969,
      "epoch": 0.4685549687950072,
      "grad_norm": 0.23046875,
      "learning_rate": 6.76253024637582e-07,
      "loss": 0.0926,
      "num_tokens": 211010612.0,
      "reward": 0.5494791939854622,
      "reward_std": 0.2048766128718853,
      "rewards/accuracy_reward": 0.14583333861082792,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.7396240234375,
      "epoch": 0.4689390302448392,
      "grad_norm": 0.2060546875,
      "learning_rate": 6.756735192801329e-07,
      "loss": 0.1115,
      "num_tokens": 211183211.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.19914305582642555,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.1771240234375,
      "epoch": 0.4693230916946711,
      "grad_norm": 0.1943359375,
      "learning_rate": 6.750937877866994e-07,
      "loss": 0.0526,
      "num_tokens": 211369708.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.18445449229329824,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1378.3021240234375,
      "epoch": 0.4697071531445031,
      "grad_norm": 0.2412109375,
      "learning_rate": 6.745138312004463e-07,
      "loss": 0.1508,
      "num_tokens": 211516809.0,
      "reward": 0.4895833432674408,
      "reward_std": 0.14156618155539036,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.4792175292969,
      "epoch": 0.4700912145943351,
      "grad_norm": 0.1787109375,
      "learning_rate": 6.739336505649427e-07,
      "loss": 0.1033,
      "num_tokens": 211693431.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.19132846593856812,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.4166870117188,
      "epoch": 0.47047527604416706,
      "grad_norm": 0.2099609375,
      "learning_rate": 6.733532469241614e-07,
      "loss": 0.1006,
      "num_tokens": 211874863.0,
      "reward": 0.5911458507180214,
      "reward_std": 0.2773894127458334,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1458.7916870117188,
      "epoch": 0.47085933749399905,
      "grad_norm": 0.236328125,
      "learning_rate": 6.727726213224766e-07,
      "loss": 0.07,
      "num_tokens": 212028763.0,
      "reward": 0.502604179084301,
      "reward_std": 0.193184956908226,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541641831398,
      "step": 1226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.0521240234375,
      "epoch": 0.471243398943831,
      "grad_norm": 0.263671875,
      "learning_rate": 6.721917748046614e-07,
      "loss": 0.113,
      "num_tokens": 212196048.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.14869255106896162,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833432674408,
      "step": 1227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1810.6458740234375,
      "epoch": 0.471627460393663,
      "grad_norm": 0.1640625,
      "learning_rate": 6.716107084158866e-07,
      "loss": 0.0655,
      "num_tokens": 212385174.0,
      "reward": 0.380208358168602,
      "reward_std": 0.1361194457858801,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583283662796,
      "step": 1228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1578.8958740234375,
      "epoch": 0.472011521843495,
      "grad_norm": 0.25,
      "learning_rate": 6.710294232017188e-07,
      "loss": 0.1455,
      "num_tokens": 212550300.0,
      "reward": 0.5208333507180214,
      "reward_std": 0.2665945179760456,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 1229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.7708740234375,
      "epoch": 0.4723955832933269,
      "grad_norm": 0.2119140625,
      "learning_rate": 6.704479202081182e-07,
      "loss": 0.1014,
      "num_tokens": 212717702.0,
      "reward": 0.6093750149011612,
      "reward_std": 0.2976357936859131,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416641831398,
      "step": 1230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.5833740234375,
      "epoch": 0.4727796447431589,
      "grad_norm": 0.216796875,
      "learning_rate": 6.69866200481437e-07,
      "loss": 0.0901,
      "num_tokens": 212874062.0,
      "reward": 0.6250000223517418,
      "reward_std": 0.35472260415554047,
      "rewards/accuracy_reward": 0.1666666716337204,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333432674408,
      "step": 1231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.2083435058594,
      "epoch": 0.47316370619299086,
      "grad_norm": 0.1982421875,
      "learning_rate": 6.692842650684172e-07,
      "loss": 0.1146,
      "num_tokens": 213054090.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.2519583459943533,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 1232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.3333740234375,
      "epoch": 0.47354776764282286,
      "grad_norm": 0.15625,
      "learning_rate": 6.687021150161889e-07,
      "loss": 0.0202,
      "num_tokens": 213242986.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.192476287484169,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.1770935058594,
      "epoch": 0.4739318290926548,
      "grad_norm": 0.1865234375,
      "learning_rate": 6.681197513722688e-07,
      "loss": 0.0811,
      "num_tokens": 213434499.0,
      "reward": 0.408854179084301,
      "reward_std": 0.1719739194959402,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1319.0833740234375,
      "epoch": 0.4743158905424868,
      "grad_norm": 0.265625,
      "learning_rate": 6.675371751845573e-07,
      "loss": 0.1185,
      "num_tokens": 213576091.0,
      "reward": 0.606770858168602,
      "reward_std": 0.3265225403010845,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4817708432674408,
      "step": 1235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1538.3750305175781,
      "epoch": 0.4746999519923188,
      "grad_norm": 0.2197265625,
      "learning_rate": 6.66954387501338e-07,
      "loss": 0.0792,
      "num_tokens": 213738759.0,
      "reward": 0.533854179084301,
      "reward_std": 0.24457025527954102,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 1236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1497.2604675292969,
      "epoch": 0.47508401344215073,
      "grad_norm": 0.2431640625,
      "learning_rate": 6.663713893712743e-07,
      "loss": 0.0907,
      "num_tokens": 213899064.0,
      "reward": 0.5598958432674408,
      "reward_std": 0.25054241716861725,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 1237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.0000305175781,
      "epoch": 0.47546807489198273,
      "grad_norm": 0.20703125,
      "learning_rate": 6.65788181843409e-07,
      "loss": 0.123,
      "num_tokens": 214079128.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.21418029069900513,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.6979522705078,
      "epoch": 0.47585213634181467,
      "grad_norm": 0.24609375,
      "learning_rate": 6.652047659671611e-07,
      "loss": 0.1054,
      "num_tokens": 214236587.0,
      "reward": 0.6119791865348816,
      "reward_std": 0.25154663249850273,
      "rewards/accuracy_reward": 0.18749999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 1239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.5208435058594,
      "epoch": 0.47623619779164666,
      "grad_norm": 0.201171875,
      "learning_rate": 6.646211427923248e-07,
      "loss": 0.0837,
      "num_tokens": 214413789.0,
      "reward": 0.479166679084301,
      "reward_std": 0.20909270457923412,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 1240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1369.0104370117188,
      "epoch": 0.47662025924147866,
      "grad_norm": 0.2392578125,
      "learning_rate": 6.640373133690671e-07,
      "loss": 0.1425,
      "num_tokens": 214559006.0,
      "reward": 0.6406250149011612,
      "reward_std": 0.26131147891283035,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.473958320915699,
      "step": 1241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.7604370117188,
      "epoch": 0.4770043206913106,
      "grad_norm": 0.2041015625,
      "learning_rate": 6.634532787479266e-07,
      "loss": 0.0413,
      "num_tokens": 214725479.0,
      "reward": 0.536458358168602,
      "reward_std": 0.209200881421566,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1762.6979370117188,
      "epoch": 0.4773883821411426,
      "grad_norm": 0.1806640625,
      "learning_rate": 6.628690399798104e-07,
      "loss": 0.1284,
      "num_tokens": 214907258.0,
      "reward": 0.4140625,
      "reward_std": 0.1886427029967308,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.5417175292969,
      "epoch": 0.47777244359097454,
      "grad_norm": 0.193359375,
      "learning_rate": 6.622845981159936e-07,
      "loss": 0.1086,
      "num_tokens": 215091574.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.3008284717798233,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.3750610351562,
      "epoch": 0.47815650504080653,
      "grad_norm": 0.2314453125,
      "learning_rate": 6.616999542081163e-07,
      "loss": 0.1445,
      "num_tokens": 215252834.0,
      "reward": 0.5468750223517418,
      "reward_std": 0.22740286216139793,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1866.3021240234375,
      "epoch": 0.4785405664906385,
      "grad_norm": 0.1611328125,
      "learning_rate": 6.611151093081827e-07,
      "loss": 0.0551,
      "num_tokens": 215445791.0,
      "reward": 0.3333333507180214,
      "reward_std": 0.07048498233780265,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333283662796,
      "step": 1246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.3541870117188,
      "epoch": 0.47892462794047047,
      "grad_norm": 0.203125,
      "learning_rate": 6.605300644685578e-07,
      "loss": 0.1139,
      "num_tokens": 215618785.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.1833493858575821,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1525.5521240234375,
      "epoch": 0.47930868939030247,
      "grad_norm": 0.220703125,
      "learning_rate": 6.599448207419675e-07,
      "loss": 0.068,
      "num_tokens": 215782830.0,
      "reward": 0.4739583507180214,
      "reward_std": 0.15532624255865812,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 1248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.0416870117188,
      "epoch": 0.4796927508401344,
      "grad_norm": 0.189453125,
      "learning_rate": 6.593593791814943e-07,
      "loss": 0.0818,
      "num_tokens": 215968306.0,
      "reward": 0.432291679084301,
      "reward_std": 0.18260713666677475,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.0625610351562,
      "epoch": 0.4800768122899664,
      "grad_norm": 0.2265625,
      "learning_rate": 6.587737408405779e-07,
      "loss": 0.0261,
      "num_tokens": 216140360.0,
      "reward": 0.4739583358168602,
      "reward_std": 0.1670458521693945,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.4791870117188,
      "epoch": 0.48046087373979834,
      "grad_norm": 0.1787109375,
      "learning_rate": 6.581879067730111e-07,
      "loss": 0.0441,
      "num_tokens": 216322806.0,
      "reward": 0.3750000149011612,
      "reward_std": 0.11365985218435526,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 1251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.7604370117188,
      "epoch": 0.48084493518963034,
      "grad_norm": 0.212890625,
      "learning_rate": 6.576018780329396e-07,
      "loss": 0.0735,
      "num_tokens": 216505535.0,
      "reward": 0.4036458507180214,
      "reward_std": 0.1313917338848114,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.3021240234375,
      "epoch": 0.48122899663946234,
      "grad_norm": 0.193359375,
      "learning_rate": 6.57015655674859e-07,
      "loss": 0.1041,
      "num_tokens": 216684412.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.17212711926549673,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291567325592,
      "step": 1253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.2604370117188,
      "epoch": 0.4816130580892943,
      "grad_norm": 0.259765625,
      "learning_rate": 6.564292407536133e-07,
      "loss": 0.1387,
      "num_tokens": 216842141.0,
      "reward": 0.6484375223517418,
      "reward_std": 0.30722833797335625,
      "rewards/accuracy_reward": 0.20833333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041567325592,
      "step": 1254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.9063110351562,
      "epoch": 0.4819971195391263,
      "grad_norm": 0.228515625,
      "learning_rate": 6.558426343243933e-07,
      "loss": 0.0646,
      "num_tokens": 217010412.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.21886225789785385,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1383.8125305175781,
      "epoch": 0.4823811809889582,
      "grad_norm": 0.349609375,
      "learning_rate": 6.552558374427338e-07,
      "loss": 0.0948,
      "num_tokens": 217154730.0,
      "reward": 0.5885416716337204,
      "reward_std": 0.3076716847717762,
      "rewards/accuracy_reward": 0.13541666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 1256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1376.5625457763672,
      "epoch": 0.4827652424387902,
      "grad_norm": 0.294921875,
      "learning_rate": 6.546688511645132e-07,
      "loss": 0.0695,
      "num_tokens": 217300504.0,
      "reward": 0.606770858168602,
      "reward_std": 0.17057401034981012,
      "rewards/accuracy_reward": 0.1458333283662796,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 1257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.3854370117188,
      "epoch": 0.4831493038886222,
      "grad_norm": 0.20703125,
      "learning_rate": 6.540816765459498e-07,
      "loss": 0.0933,
      "num_tokens": 217465253.0,
      "reward": 0.5572916865348816,
      "reward_std": 0.2883218778297305,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1673.0521240234375,
      "epoch": 0.48353336533845415,
      "grad_norm": 0.208984375,
      "learning_rate": 6.534943146436015e-07,
      "loss": 0.0533,
      "num_tokens": 217642658.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.21063288860023022,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875074505806,
      "step": 1259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.2188110351562,
      "epoch": 0.48391742678828614,
      "grad_norm": 0.25390625,
      "learning_rate": 6.529067665143625e-07,
      "loss": 0.1421,
      "num_tokens": 217796927.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.17973750457167625,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.3958740234375,
      "epoch": 0.4843014882381181,
      "grad_norm": 0.2099609375,
      "learning_rate": 6.523190332154629e-07,
      "loss": 0.0198,
      "num_tokens": 217964533.0,
      "reward": 0.5130208358168602,
      "reward_std": 0.22337975725531578,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.6354370117188,
      "epoch": 0.4846855496879501,
      "grad_norm": 0.201171875,
      "learning_rate": 6.517311158044651e-07,
      "loss": 0.1324,
      "num_tokens": 218131922.0,
      "reward": 0.5130208656191826,
      "reward_std": 0.2114442065358162,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 1262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.3333740234375,
      "epoch": 0.485069611137782,
      "grad_norm": 0.2373046875,
      "learning_rate": 6.511430153392638e-07,
      "loss": 0.1112,
      "num_tokens": 218300514.0,
      "reward": 0.4791666716337204,
      "reward_std": 0.25834412686526775,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.416666679084301,
      "step": 1263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.0625305175781,
      "epoch": 0.485453672587614,
      "grad_norm": 0.2119140625,
      "learning_rate": 6.505547328780822e-07,
      "loss": 0.1334,
      "num_tokens": 218465824.0,
      "reward": 0.5859375149011612,
      "reward_std": 0.30452491343021393,
      "rewards/accuracy_reward": 0.16666666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.7396545410156,
      "epoch": 0.485837734037446,
      "grad_norm": 0.224609375,
      "learning_rate": 6.499662694794714e-07,
      "loss": 0.1008,
      "num_tokens": 218640871.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.21989169344305992,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.0937805175781,
      "epoch": 0.48622179548727795,
      "grad_norm": 0.2197265625,
      "learning_rate": 6.493776262023079e-07,
      "loss": 0.1186,
      "num_tokens": 218814720.0,
      "reward": 0.4817708432674408,
      "reward_std": 0.23100525047630072,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.1979675292969,
      "epoch": 0.48660585693710995,
      "grad_norm": 0.1962890625,
      "learning_rate": 6.487888041057923e-07,
      "loss": 0.082,
      "num_tokens": 218992531.0,
      "reward": 0.497395858168602,
      "reward_std": 0.24970513535663486,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.3437805175781,
      "epoch": 0.4869899183869419,
      "grad_norm": 0.23046875,
      "learning_rate": 6.481998042494464e-07,
      "loss": 0.1197,
      "num_tokens": 219154268.0,
      "reward": 0.466145858168602,
      "reward_std": 0.17948868498206139,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.9895935058594,
      "epoch": 0.4873739798367739,
      "grad_norm": 0.2099609375,
      "learning_rate": 6.476106276931122e-07,
      "loss": 0.0936,
      "num_tokens": 219333699.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.2443016916513443,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3750000074505806,
      "step": 1269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.2187805175781,
      "epoch": 0.4877580412866059,
      "grad_norm": 0.28125,
      "learning_rate": 6.470212754969493e-07,
      "loss": 0.0495,
      "num_tokens": 219498704.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.2265063039958477,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.6771240234375,
      "epoch": 0.4881421027364378,
      "grad_norm": 0.1884765625,
      "learning_rate": 6.46431748721434e-07,
      "loss": 0.0976,
      "num_tokens": 219688817.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.19719426333904266,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 1271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.2292175292969,
      "epoch": 0.4885261641862698,
      "grad_norm": 0.236328125,
      "learning_rate": 6.458420484273563e-07,
      "loss": 0.0914,
      "num_tokens": 219857487.0,
      "reward": 0.4322916716337204,
      "reward_std": 0.16045633517205715,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.5313110351562,
      "epoch": 0.48891022563610176,
      "grad_norm": 0.2138671875,
      "learning_rate": 6.45252175675818e-07,
      "loss": 0.0781,
      "num_tokens": 220039194.0,
      "reward": 0.5000000074505806,
      "reward_std": 0.24222352914512157,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.6667175292969,
      "epoch": 0.48929428708593375,
      "grad_norm": 0.2294921875,
      "learning_rate": 6.446621315282322e-07,
      "loss": 0.0932,
      "num_tokens": 220212298.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.21087615564465523,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.4167175292969,
      "epoch": 0.4896783485357657,
      "grad_norm": 0.20703125,
      "learning_rate": 6.440719170463196e-07,
      "loss": 0.0814,
      "num_tokens": 220379778.0,
      "reward": 0.5078125074505806,
      "reward_std": 0.26913951337337494,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 1275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1498.5000305175781,
      "epoch": 0.4900624099855977,
      "grad_norm": 0.271484375,
      "learning_rate": 6.43481533292108e-07,
      "loss": 0.1636,
      "num_tokens": 220537514.0,
      "reward": 0.5859375,
      "reward_std": 0.3285258263349533,
      "rewards/accuracy_reward": 0.12500000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 1276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.729248046875,
      "epoch": 0.4904464714354297,
      "grad_norm": 0.21484375,
      "learning_rate": 6.428909813279289e-07,
      "loss": 0.0775,
      "num_tokens": 220710568.0,
      "reward": 0.4895833358168602,
      "reward_std": 0.23828747496008873,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 1277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1502.3229675292969,
      "epoch": 0.49083053288526163,
      "grad_norm": 0.20703125,
      "learning_rate": 6.423002622164178e-07,
      "loss": 0.0538,
      "num_tokens": 220869615.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.12491247244179249,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.9583740234375,
      "epoch": 0.4912145943350936,
      "grad_norm": 0.2197265625,
      "learning_rate": 6.417093770205098e-07,
      "loss": 0.0989,
      "num_tokens": 221041051.0,
      "reward": 0.5364583358168602,
      "reward_std": 0.23975254502147436,
      "rewards/accuracy_reward": 0.16666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.4062805175781,
      "epoch": 0.49159865578492556,
      "grad_norm": 0.20703125,
      "learning_rate": 6.411183268034393e-07,
      "loss": 0.0622,
      "num_tokens": 221219786.0,
      "reward": 0.4322916865348816,
      "reward_std": 0.18061827775090933,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416567325592,
      "step": 1280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.2812805175781,
      "epoch": 0.49198271723475756,
      "grad_norm": 0.2109375,
      "learning_rate": 6.405271126287376e-07,
      "loss": 0.1068,
      "num_tokens": 221404157.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.18520670011639595,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1392.0000610351562,
      "epoch": 0.49236677868458956,
      "grad_norm": 0.294921875,
      "learning_rate": 6.399357355602312e-07,
      "loss": 0.0768,
      "num_tokens": 221550909.0,
      "reward": 0.6171875074505806,
      "reward_std": 0.2703980356454849,
      "rewards/accuracy_reward": 0.12500000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875074505806,
      "step": 1282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1699.7604675292969,
      "epoch": 0.4927508401344215,
      "grad_norm": 0.228515625,
      "learning_rate": 6.393441966620394e-07,
      "loss": 0.104,
      "num_tokens": 221730814.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.2292390838265419,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140624925494194,
      "step": 1283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1427.3541870117188,
      "epoch": 0.4931349015842535,
      "grad_norm": 0.279296875,
      "learning_rate": 6.387524969985728e-07,
      "loss": 0.1308,
      "num_tokens": 221880824.0,
      "reward": 0.5364583507180214,
      "reward_std": 0.18103139754384756,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 1284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1518.9687805175781,
      "epoch": 0.49351896303408543,
      "grad_norm": 0.2265625,
      "learning_rate": 6.381606376345315e-07,
      "loss": 0.0983,
      "num_tokens": 222040229.0,
      "reward": 0.5520833358168602,
      "reward_std": 0.26577775180339813,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 1285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1699.7916870117188,
      "epoch": 0.49390302448391743,
      "grad_norm": 0.2021484375,
      "learning_rate": 6.375686196349026e-07,
      "loss": 0.0606,
      "num_tokens": 222219081.0,
      "reward": 0.432291679084301,
      "reward_std": 0.18822155334055424,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.2291870117188,
      "epoch": 0.4942870859337494,
      "grad_norm": 0.2236328125,
      "learning_rate": 6.369764440649588e-07,
      "loss": 0.1308,
      "num_tokens": 222395919.0,
      "reward": 0.3828125074505806,
      "reward_std": 0.12857080437242985,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.9062805175781,
      "epoch": 0.49467114738358137,
      "grad_norm": 0.1865234375,
      "learning_rate": 6.363841119902566e-07,
      "loss": 0.1009,
      "num_tokens": 222581094.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.16945991991087794,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1477.5104370117188,
      "epoch": 0.49505520883341336,
      "grad_norm": 0.306640625,
      "learning_rate": 6.357916244766337e-07,
      "loss": 0.0222,
      "num_tokens": 222741023.0,
      "reward": 0.5781250149011612,
      "reward_std": 0.27867643535137177,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083432674408,
      "step": 1289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.2187805175781,
      "epoch": 0.4954392702832453,
      "grad_norm": 0.1962890625,
      "learning_rate": 6.351989825902078e-07,
      "loss": 0.1052,
      "num_tokens": 222919956.0,
      "reward": 0.471354179084301,
      "reward_std": 0.2602863796055317,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1348.2083740234375,
      "epoch": 0.4958233317330773,
      "grad_norm": 0.244140625,
      "learning_rate": 6.34606187397374e-07,
      "loss": 0.1217,
      "num_tokens": 223064032.0,
      "reward": 0.546875,
      "reward_std": 0.26376965641975403,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583358168602,
      "step": 1291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1389.8750305175781,
      "epoch": 0.49620739318290924,
      "grad_norm": 0.2470703125,
      "learning_rate": 6.340132399648036e-07,
      "loss": 0.0428,
      "num_tokens": 223211844.0,
      "reward": 0.7447916865348816,
      "reward_std": 0.2803783752024174,
      "rewards/accuracy_reward": 0.2916666669771075,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531249925494194,
      "step": 1292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.8750610351562,
      "epoch": 0.49659145463274124,
      "grad_norm": 0.1923828125,
      "learning_rate": 6.334201413594421e-07,
      "loss": 0.0961,
      "num_tokens": 223375464.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.17862650100141764,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 1293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.2395935058594,
      "epoch": 0.49697551608257323,
      "grad_norm": 0.197265625,
      "learning_rate": 6.328268926485059e-07,
      "loss": 0.0833,
      "num_tokens": 223559439.0,
      "reward": 0.5338541716337204,
      "reward_std": 0.2657053843140602,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.0729370117188,
      "epoch": 0.4973595775324052,
      "grad_norm": 0.2021484375,
      "learning_rate": 6.322334948994831e-07,
      "loss": 0.0791,
      "num_tokens": 223731294.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.15041559003293514,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 1295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1579.6875610351562,
      "epoch": 0.49774363898223717,
      "grad_norm": 0.271484375,
      "learning_rate": 6.316399491801288e-07,
      "loss": 0.1384,
      "num_tokens": 223897592.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.20831861160695553,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.8437805175781,
      "epoch": 0.4981277004320691,
      "grad_norm": 0.1953125,
      "learning_rate": 6.310462565584647e-07,
      "loss": 0.1691,
      "num_tokens": 224073481.0,
      "reward": 0.4921875223517418,
      "reward_std": 0.2698871102184057,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 1297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.5625305175781,
      "epoch": 0.4985117618819011,
      "grad_norm": 0.1904296875,
      "learning_rate": 6.304524181027765e-07,
      "loss": 0.105,
      "num_tokens": 224258999.0,
      "reward": 0.4140625,
      "reward_std": 0.20122049562633038,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 1298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1537.6771545410156,
      "epoch": 0.4988958233317331,
      "grad_norm": 0.2451171875,
      "learning_rate": 6.298584348816133e-07,
      "loss": 0.1016,
      "num_tokens": 224422224.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.282752089202404,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1334.4062957763672,
      "epoch": 0.49927988478156504,
      "grad_norm": 0.2734375,
      "learning_rate": 6.292643079637834e-07,
      "loss": 0.0758,
      "num_tokens": 224564895.0,
      "reward": 0.6796875149011612,
      "reward_std": 0.31592296808958054,
      "rewards/accuracy_reward": 0.19791666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4817708283662796,
      "step": 1300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.0312805175781,
      "epoch": 0.49966394623139704,
      "grad_norm": 0.20703125,
      "learning_rate": 6.286700384183547e-07,
      "loss": 0.0605,
      "num_tokens": 224743250.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.15529664978384972,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083432674408,
      "step": 1301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.5729675292969,
      "epoch": 0.500048007681229,
      "grad_norm": 0.21484375,
      "learning_rate": 6.280756273146512e-07,
      "loss": 0.146,
      "num_tokens": 224903641.0,
      "reward": 0.5260416716337204,
      "reward_std": 0.2653418220579624,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416641831398,
      "step": 1302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1823.2291870117188,
      "epoch": 0.500432069131061,
      "grad_norm": 0.1904296875,
      "learning_rate": 6.274810757222521e-07,
      "loss": 0.0994,
      "num_tokens": 225091383.0,
      "reward": 0.3802083507180214,
      "reward_std": 0.15876208990812302,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 1303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.979248046875,
      "epoch": 0.5008161305808929,
      "grad_norm": 0.1904296875,
      "learning_rate": 6.268863847109883e-07,
      "loss": 0.1036,
      "num_tokens": 225263037.0,
      "reward": 0.3854166716337204,
      "reward_std": 0.12011965736746788,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.1146240234375,
      "epoch": 0.501200192030725,
      "grad_norm": 0.1923828125,
      "learning_rate": 6.262915553509431e-07,
      "loss": 0.099,
      "num_tokens": 225441288.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.13921440951526165,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 1305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.5938110351562,
      "epoch": 0.5015842534805569,
      "grad_norm": 0.1943359375,
      "learning_rate": 6.256965887124476e-07,
      "loss": 0.0991,
      "num_tokens": 225606305.0,
      "reward": 0.4713541865348816,
      "reward_std": 0.20861471816897392,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1297.7813110351562,
      "epoch": 0.5019683149303888,
      "grad_norm": 0.32421875,
      "learning_rate": 6.251014858660802e-07,
      "loss": 0.0933,
      "num_tokens": 225743316.0,
      "reward": 0.565104179084301,
      "reward_std": 0.2202686332166195,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875,
      "step": 1307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1367.9167175292969,
      "epoch": 0.5023523763802208,
      "grad_norm": 0.2490234375,
      "learning_rate": 6.245062478826647e-07,
      "loss": 0.1268,
      "num_tokens": 225887724.0,
      "reward": 0.6119791865348816,
      "reward_std": 0.28791364654898643,
      "rewards/accuracy_reward": 0.12500000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4869791716337204,
      "step": 1308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.2708740234375,
      "epoch": 0.5027364378300528,
      "grad_norm": 0.2099609375,
      "learning_rate": 6.239108758332678e-07,
      "loss": 0.0741,
      "num_tokens": 226069030.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.2032490111887455,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 1309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.7188110351562,
      "epoch": 0.5031204992798848,
      "grad_norm": 0.1728515625,
      "learning_rate": 6.233153707891976e-07,
      "loss": 0.0597,
      "num_tokens": 226253107.0,
      "reward": 0.3437500074505806,
      "reward_std": 0.08995514642447233,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3437500074505806,
      "step": 1310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1778.5208740234375,
      "epoch": 0.5035045607297167,
      "grad_norm": 0.1953125,
      "learning_rate": 6.22719733822001e-07,
      "loss": 0.0757,
      "num_tokens": 226439293.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.21145523339509964,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.3021240234375,
      "epoch": 0.5038886221795488,
      "grad_norm": 0.267578125,
      "learning_rate": 6.221239660034631e-07,
      "loss": 0.1305,
      "num_tokens": 226601794.0,
      "reward": 0.494791679084301,
      "reward_std": 0.227768462151289,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 1312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1244.8125457763672,
      "epoch": 0.5042726836293807,
      "grad_norm": 0.25,
      "learning_rate": 6.215280684056037e-07,
      "loss": 0.1103,
      "num_tokens": 226733088.0,
      "reward": 0.5520833507180214,
      "reward_std": 0.17266854643821716,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4895833283662796,
      "step": 1313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.5417175292969,
      "epoch": 0.5046567450792127,
      "grad_norm": 0.251953125,
      "learning_rate": 6.209320421006768e-07,
      "loss": 0.0903,
      "num_tokens": 226895484.0,
      "reward": 0.4635416716337204,
      "reward_std": 0.1931825652718544,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 1314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.78125,
      "epoch": 0.5050408065290446,
      "grad_norm": 0.1982421875,
      "learning_rate": 6.203358881611674e-07,
      "loss": 0.0987,
      "num_tokens": 227074143.0,
      "reward": 0.518229179084301,
      "reward_std": 0.2836921475827694,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1590.0729675292969,
      "epoch": 0.5054248679788766,
      "grad_norm": 0.2373046875,
      "learning_rate": 6.197396076597902e-07,
      "loss": 0.0994,
      "num_tokens": 227243070.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.17283223569393158,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 1316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.5209045410156,
      "epoch": 0.5058089294287086,
      "grad_norm": 0.25390625,
      "learning_rate": 6.191432016694884e-07,
      "loss": 0.1689,
      "num_tokens": 227414576.0,
      "reward": 0.559895858168602,
      "reward_std": 0.28936753794550896,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.0521545410156,
      "epoch": 0.5061929908785405,
      "grad_norm": 0.2001953125,
      "learning_rate": 6.185466712634301e-07,
      "loss": 0.1317,
      "num_tokens": 227599501.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.15430257469415665,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.0312805175781,
      "epoch": 0.5065770523283726,
      "grad_norm": 0.228515625,
      "learning_rate": 6.179500175150076e-07,
      "loss": 0.1249,
      "num_tokens": 227776400.0,
      "reward": 0.466145858168602,
      "reward_std": 0.2744695171713829,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.0833740234375,
      "epoch": 0.5069611137782045,
      "grad_norm": 0.21484375,
      "learning_rate": 6.173532414978352e-07,
      "loss": 0.0441,
      "num_tokens": 227958328.0,
      "reward": 0.3750000149011612,
      "reward_std": 0.11579695157706738,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 1320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1526.8750610351562,
      "epoch": 0.5073451752280365,
      "grad_norm": 0.263671875,
      "learning_rate": 6.167563442857472e-07,
      "loss": 0.2137,
      "num_tokens": 228119836.0,
      "reward": 0.5729166716337204,
      "reward_std": 0.30312035232782364,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 1321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1899.3333740234375,
      "epoch": 0.5077292366778685,
      "grad_norm": 0.2158203125,
      "learning_rate": 6.161593269527959e-07,
      "loss": 0.0848,
      "num_tokens": 228322284.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.2201901338994503,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416641831398,
      "step": 1322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.3021240234375,
      "epoch": 0.5081132981277005,
      "grad_norm": 0.2216796875,
      "learning_rate": 6.1556219057325e-07,
      "loss": 0.1306,
      "num_tokens": 228492089.0,
      "reward": 0.4661458507180214,
      "reward_std": 0.21563363634049892,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140624925494194,
      "step": 1323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.7604675292969,
      "epoch": 0.5084973595775324,
      "grad_norm": 0.2119140625,
      "learning_rate": 6.14964936221592e-07,
      "loss": 0.0594,
      "num_tokens": 228675858.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.18820533901453018,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.8854370117188,
      "epoch": 0.5088814210273643,
      "grad_norm": 0.271484375,
      "learning_rate": 6.143675649725169e-07,
      "loss": 0.0471,
      "num_tokens": 228831631.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.16366158332675695,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 1325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1677.1458740234375,
      "epoch": 0.5092654824771964,
      "grad_norm": 0.2138671875,
      "learning_rate": 6.1377007790093e-07,
      "loss": 0.0805,
      "num_tokens": 229007149.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.20062123239040375,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 1326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1405.2708740234375,
      "epoch": 0.5096495439270283,
      "grad_norm": 0.287109375,
      "learning_rate": 6.131724760819453e-07,
      "loss": 0.0567,
      "num_tokens": 229153951.0,
      "reward": 0.4375000223517418,
      "reward_std": 0.14008726086467505,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.427083320915699,
      "step": 1327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1379.3854370117188,
      "epoch": 0.5100336053768603,
      "grad_norm": 0.2138671875,
      "learning_rate": 6.125747605908827e-07,
      "loss": 0.0801,
      "num_tokens": 229297668.0,
      "reward": 0.5807291939854622,
      "reward_std": 0.24005621951073408,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 1328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.1666870117188,
      "epoch": 0.5104176668266923,
      "grad_norm": 0.2265625,
      "learning_rate": 6.11976932503267e-07,
      "loss": 0.1045,
      "num_tokens": 229455508.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.21186734549701214,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 1329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1598.6250305175781,
      "epoch": 0.5108017282765243,
      "grad_norm": 0.228515625,
      "learning_rate": 6.113789928948256e-07,
      "loss": 0.086,
      "num_tokens": 229625640.0,
      "reward": 0.5729166939854622,
      "reward_std": 0.2625688537955284,
      "rewards/accuracy_reward": 0.14583333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1495.2708740234375,
      "epoch": 0.5111857897263562,
      "grad_norm": 0.2314453125,
      "learning_rate": 6.107809428414867e-07,
      "loss": 0.1155,
      "num_tokens": 229783666.0,
      "reward": 0.5390625298023224,
      "reward_std": 0.23502438887953758,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.0417175292969,
      "epoch": 0.5115698511761881,
      "grad_norm": 0.1826171875,
      "learning_rate": 6.101827834193768e-07,
      "loss": 0.0973,
      "num_tokens": 229974990.0,
      "reward": 0.4036458507180214,
      "reward_std": 0.20586001314222813,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.6250610351562,
      "epoch": 0.5119539126260202,
      "grad_norm": 0.234375,
      "learning_rate": 6.095845157048197e-07,
      "loss": 0.1121,
      "num_tokens": 230155258.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.16286470368504524,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1491.5625305175781,
      "epoch": 0.5123379740758521,
      "grad_norm": 0.25,
      "learning_rate": 6.089861407743336e-07,
      "loss": 0.1739,
      "num_tokens": 230312472.0,
      "reward": 0.502604179084301,
      "reward_std": 0.24630161374807358,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 1334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.1250610351562,
      "epoch": 0.5127220355256841,
      "grad_norm": 0.2216796875,
      "learning_rate": 6.0838765970463e-07,
      "loss": 0.0691,
      "num_tokens": 230494316.0,
      "reward": 0.4661458507180214,
      "reward_std": 0.20555372908711433,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.5937805175781,
      "epoch": 0.5131060969755161,
      "grad_norm": 0.20703125,
      "learning_rate": 6.07789073572611e-07,
      "loss": 0.1017,
      "num_tokens": 230672325.0,
      "reward": 0.4661458507180214,
      "reward_std": 0.2741381525993347,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.6250610351562,
      "epoch": 0.5134901584253481,
      "grad_norm": 0.244140625,
      "learning_rate": 6.071903834553683e-07,
      "loss": 0.1684,
      "num_tokens": 230846321.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.1639205813407898,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.5000305175781,
      "epoch": 0.51387421987518,
      "grad_norm": 0.244140625,
      "learning_rate": 6.065915904301801e-07,
      "loss": 0.1371,
      "num_tokens": 231005857.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.2936718873679638,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875074505806,
      "step": 1338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1231.0521240234375,
      "epoch": 0.5142582813250121,
      "grad_norm": 0.296875,
      "learning_rate": 6.059926955745104e-07,
      "loss": 0.1272,
      "num_tokens": 231136510.0,
      "reward": 0.6015625298023224,
      "reward_std": 0.28047940135002136,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4973958358168602,
      "step": 1339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.2500305175781,
      "epoch": 0.514642342774844,
      "grad_norm": 0.216796875,
      "learning_rate": 6.053936999660058e-07,
      "loss": 0.1069,
      "num_tokens": 231304910.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.24024224281311035,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 1340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.3854675292969,
      "epoch": 0.5150264042246759,
      "grad_norm": 0.2373046875,
      "learning_rate": 6.047946046824947e-07,
      "loss": 0.1186,
      "num_tokens": 231475971.0,
      "reward": 0.5364583507180214,
      "reward_std": 0.2381509467959404,
      "rewards/accuracy_reward": 0.1458333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.3021240234375,
      "epoch": 0.5154104656745079,
      "grad_norm": 0.2470703125,
      "learning_rate": 6.041954108019845e-07,
      "loss": 0.145,
      "num_tokens": 231640240.0,
      "reward": 0.440104179084301,
      "reward_std": 0.18094799853861332,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 1342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.447998046875,
      "epoch": 0.5157945271243399,
      "grad_norm": 0.23046875,
      "learning_rate": 6.0359611940266e-07,
      "loss": 0.0716,
      "num_tokens": 231815651.0,
      "reward": 0.4661458507180214,
      "reward_std": 0.20265774801373482,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.9375305175781,
      "epoch": 0.5161785885741719,
      "grad_norm": 0.2392578125,
      "learning_rate": 6.02996731562882e-07,
      "loss": 0.0663,
      "num_tokens": 231992077.0,
      "reward": 0.3750000074505806,
      "reward_std": 0.12437989562749863,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.0729370117188,
      "epoch": 0.5165626500240038,
      "grad_norm": 0.21484375,
      "learning_rate": 6.023972483611842e-07,
      "loss": 0.1375,
      "num_tokens": 232153356.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.1780752446502447,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.1771240234375,
      "epoch": 0.5169467114738359,
      "grad_norm": 0.1982421875,
      "learning_rate": 6.017976708762723e-07,
      "loss": 0.1203,
      "num_tokens": 232327093.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.16930270195007324,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 1346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1667.7187805175781,
      "epoch": 0.5173307729236678,
      "grad_norm": 0.1884765625,
      "learning_rate": 6.011980001870214e-07,
      "loss": 0.0704,
      "num_tokens": 232503130.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.19866715371608734,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 1347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.0000305175781,
      "epoch": 0.5177148343734997,
      "grad_norm": 0.173828125,
      "learning_rate": 6.005982373724743e-07,
      "loss": 0.1055,
      "num_tokens": 232681138.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.23989990446716547,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.2083740234375,
      "epoch": 0.5180988958233317,
      "grad_norm": 0.158203125,
      "learning_rate": 5.9999838351184e-07,
      "loss": 0.0612,
      "num_tokens": 232864622.0,
      "reward": 0.4166666865348816,
      "reward_std": 0.15746252425014973,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166567325592,
      "step": 1349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.4687805175781,
      "epoch": 0.5184829572731637,
      "grad_norm": 0.216796875,
      "learning_rate": 5.993984396844911e-07,
      "loss": 0.09,
      "num_tokens": 233036347.0,
      "reward": 0.486979179084301,
      "reward_std": 0.16486968845129013,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 1350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.8854370117188,
      "epoch": 0.5188670187229957,
      "grad_norm": 0.2158203125,
      "learning_rate": 5.987984069699614e-07,
      "loss": 0.1311,
      "num_tokens": 233199328.0,
      "reward": 0.5104166716337204,
      "reward_std": 0.2453520204871893,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.0313110351562,
      "epoch": 0.5192510801728276,
      "grad_norm": 0.21875,
      "learning_rate": 5.98198286447946e-07,
      "loss": 0.0836,
      "num_tokens": 233358027.0,
      "reward": 0.6302083507180214,
      "reward_std": 0.303579967468977,
      "rewards/accuracy_reward": 0.16666666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416641831398,
      "step": 1352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.4583740234375,
      "epoch": 0.5196351416226597,
      "grad_norm": 0.216796875,
      "learning_rate": 5.975980791982967e-07,
      "loss": 0.0864,
      "num_tokens": 233537239.0,
      "reward": 0.526041679084301,
      "reward_std": 0.2575688026845455,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1709.6666870117188,
      "epoch": 0.5200192030724916,
      "grad_norm": 0.20703125,
      "learning_rate": 5.969977863010221e-07,
      "loss": 0.0814,
      "num_tokens": 233713943.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.20029382407665253,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1357.7188110351562,
      "epoch": 0.5204032645223235,
      "grad_norm": 0.26171875,
      "learning_rate": 5.963974088362847e-07,
      "loss": 0.1327,
      "num_tokens": 233857852.0,
      "reward": 0.5338541865348816,
      "reward_std": 0.22578012570738792,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875,
      "step": 1355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1393.3542175292969,
      "epoch": 0.5207873259721555,
      "grad_norm": 0.2392578125,
      "learning_rate": 5.957969478843993e-07,
      "loss": 0.0973,
      "num_tokens": 234007158.0,
      "reward": 0.6041667014360428,
      "reward_std": 0.3077751398086548,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 1356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1506.6354370117188,
      "epoch": 0.5211713874219875,
      "grad_norm": 0.2333984375,
      "learning_rate": 5.951964045258306e-07,
      "loss": 0.1204,
      "num_tokens": 234165707.0,
      "reward": 0.494791679084301,
      "reward_std": 0.20118052139878273,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.0729370117188,
      "epoch": 0.5215554488718195,
      "grad_norm": 0.2314453125,
      "learning_rate": 5.94595779841192e-07,
      "loss": 0.0091,
      "num_tokens": 234321402.0,
      "reward": 0.5598958507180214,
      "reward_std": 0.30084624886512756,
      "rewards/accuracy_reward": 0.10416666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291641831398,
      "step": 1358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1790.8229675292969,
      "epoch": 0.5219395103216514,
      "grad_norm": 0.1806640625,
      "learning_rate": 5.939950749112429e-07,
      "loss": 0.0637,
      "num_tokens": 234509409.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.174962074495852,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 1359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.7187805175781,
      "epoch": 0.5223235717714835,
      "grad_norm": 0.1953125,
      "learning_rate": 5.933942908168873e-07,
      "loss": 0.0885,
      "num_tokens": 234689382.0,
      "reward": 0.4036458507180214,
      "reward_std": 0.16203317791223526,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828124925494194,
      "step": 1360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.0937805175781,
      "epoch": 0.5227076332213154,
      "grad_norm": 0.20703125,
      "learning_rate": 5.927934286391713e-07,
      "loss": 0.0585,
      "num_tokens": 234857079.0,
      "reward": 0.5755208432674408,
      "reward_std": 0.2095536421984434,
      "rewards/accuracy_reward": 0.16666666232049465,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.4583435058594,
      "epoch": 0.5230916946711474,
      "grad_norm": 0.25,
      "learning_rate": 5.921924894592825e-07,
      "loss": 0.0545,
      "num_tokens": 235024459.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.18921396881341934,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.4479370117188,
      "epoch": 0.5234757561209794,
      "grad_norm": 0.2470703125,
      "learning_rate": 5.915914743585457e-07,
      "loss": 0.0857,
      "num_tokens": 235200390.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.13175934925675392,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1323.7917022705078,
      "epoch": 0.5238598175708113,
      "grad_norm": 0.287109375,
      "learning_rate": 5.909903844184229e-07,
      "loss": 0.087,
      "num_tokens": 235340274.0,
      "reward": 0.5000000223517418,
      "reward_std": 0.22025199979543686,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687500074505806,
      "step": 1364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1528.3020935058594,
      "epoch": 0.5242438790206433,
      "grad_norm": 0.208984375,
      "learning_rate": 5.903892207205113e-07,
      "loss": 0.0798,
      "num_tokens": 235500791.0,
      "reward": 0.5833333432674408,
      "reward_std": 0.2975548058748245,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 1365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.5312805175781,
      "epoch": 0.5246279404704752,
      "grad_norm": 0.1826171875,
      "learning_rate": 5.897879843465399e-07,
      "loss": 0.1169,
      "num_tokens": 235685770.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.14852811768651009,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1756.2500305175781,
      "epoch": 0.5250120019203073,
      "grad_norm": 0.1689453125,
      "learning_rate": 5.89186676378369e-07,
      "loss": 0.0634,
      "num_tokens": 235867754.0,
      "reward": 0.46875,
      "reward_std": 0.22341016680002213,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 1367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.0417175292969,
      "epoch": 0.5253960633701392,
      "grad_norm": 0.197265625,
      "learning_rate": 5.885852978979877e-07,
      "loss": 0.0872,
      "num_tokens": 236049054.0,
      "reward": 0.4114583507180214,
      "reward_std": 0.14076979411765933,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 1368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.4792175292969,
      "epoch": 0.5257801248199712,
      "grad_norm": 0.20703125,
      "learning_rate": 5.879838499875121e-07,
      "loss": 0.0808,
      "num_tokens": 236233100.0,
      "reward": 0.520833358168602,
      "reward_std": 0.2869931310415268,
      "rewards/accuracy_reward": 0.10416666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1704.7604675292969,
      "epoch": 0.5261641862698032,
      "grad_norm": 0.1826171875,
      "learning_rate": 5.873823337291827e-07,
      "loss": 0.0736,
      "num_tokens": 236411301.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.17257465794682503,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.0729675292969,
      "epoch": 0.5265482477196352,
      "grad_norm": 0.18359375,
      "learning_rate": 5.867807502053635e-07,
      "loss": 0.0762,
      "num_tokens": 236575116.0,
      "reward": 0.440104179084301,
      "reward_std": 0.17047201097011566,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.6979675292969,
      "epoch": 0.5269323091694671,
      "grad_norm": 0.2021484375,
      "learning_rate": 5.861791004985392e-07,
      "loss": 0.0909,
      "num_tokens": 236736327.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.17507965816184878,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.0104217529297,
      "epoch": 0.527316370619299,
      "grad_norm": 0.203125,
      "learning_rate": 5.855773856913141e-07,
      "loss": 0.0351,
      "num_tokens": 236911496.0,
      "reward": 0.4635416865348816,
      "reward_std": 0.18102153204381466,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 1373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1431.5312805175781,
      "epoch": 0.5277004320691311,
      "grad_norm": 0.224609375,
      "learning_rate": 5.84975606866409e-07,
      "loss": 0.1199,
      "num_tokens": 237061131.0,
      "reward": 0.5182291716337204,
      "reward_std": 0.20703357830643654,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765624925494194,
      "step": 1374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.6458740234375,
      "epoch": 0.528084493518963,
      "grad_norm": 0.2333984375,
      "learning_rate": 5.843737651066603e-07,
      "loss": 0.1595,
      "num_tokens": 237243289.0,
      "reward": 0.369791679084301,
      "reward_std": 0.14777947217226028,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.0625610351562,
      "epoch": 0.528468554968795,
      "grad_norm": 0.255859375,
      "learning_rate": 5.837718614950174e-07,
      "loss": 0.1231,
      "num_tokens": 237413799.0,
      "reward": 0.549479179084301,
      "reward_std": 0.2513149194419384,
      "rewards/accuracy_reward": 0.14583333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.5625610351562,
      "epoch": 0.528852616418627,
      "grad_norm": 0.224609375,
      "learning_rate": 5.831698971145414e-07,
      "loss": 0.0937,
      "num_tokens": 237596397.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.2573247477412224,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.1563110351562,
      "epoch": 0.529236677868459,
      "grad_norm": 0.2431640625,
      "learning_rate": 5.82567873048402e-07,
      "loss": 0.1335,
      "num_tokens": 237757828.0,
      "reward": 0.4296875223517418,
      "reward_std": 0.16225312650203705,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 1378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1707.3437805175781,
      "epoch": 0.5296207393182909,
      "grad_norm": 0.205078125,
      "learning_rate": 5.819657903798773e-07,
      "loss": 0.0319,
      "num_tokens": 237937229.0,
      "reward": 0.5677083432674408,
      "reward_std": 0.26018374040722847,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.2604370117188,
      "epoch": 0.530004800768123,
      "grad_norm": 0.1884765625,
      "learning_rate": 5.8136365019235e-07,
      "loss": 0.1102,
      "num_tokens": 238107214.0,
      "reward": 0.4843750074505806,
      "reward_std": 0.18705198680981994,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.5729675292969,
      "epoch": 0.5303888622179549,
      "grad_norm": 0.224609375,
      "learning_rate": 5.807614535693066e-07,
      "loss": 0.1754,
      "num_tokens": 238275725.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.14688404649496078,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 1381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.9271240234375,
      "epoch": 0.5307729236677868,
      "grad_norm": 0.1982421875,
      "learning_rate": 5.801592015943354e-07,
      "loss": 0.108,
      "num_tokens": 238445198.0,
      "reward": 0.5963541865348816,
      "reward_std": 0.2617654874920845,
      "rewards/accuracy_reward": 0.17708333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 1382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.2917175292969,
      "epoch": 0.5311569851176188,
      "grad_norm": 0.228515625,
      "learning_rate": 5.79556895351124e-07,
      "loss": 0.1377,
      "num_tokens": 238612962.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.25260502472519875,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1742.0312805175781,
      "epoch": 0.5315410465674508,
      "grad_norm": 0.1796875,
      "learning_rate": 5.789545359234576e-07,
      "loss": 0.1156,
      "num_tokens": 238795621.0,
      "reward": 0.432291679084301,
      "reward_std": 0.11123901512473822,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 1384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.1563110351562,
      "epoch": 0.5319251080172828,
      "grad_norm": 0.19140625,
      "learning_rate": 5.783521243952172e-07,
      "loss": 0.1056,
      "num_tokens": 238984276.0,
      "reward": 0.3776041716337204,
      "reward_std": 0.1738903746008873,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541641831398,
      "step": 1385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1484.4271240234375,
      "epoch": 0.5323091694671147,
      "grad_norm": 0.2392578125,
      "learning_rate": 5.777496618503778e-07,
      "loss": 0.1052,
      "num_tokens": 239144373.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.23372026346623898,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083432674408,
      "step": 1386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.9062805175781,
      "epoch": 0.5326932309169468,
      "grad_norm": 0.1884765625,
      "learning_rate": 5.771471493730057e-07,
      "loss": 0.0876,
      "num_tokens": 239317220.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.18028570897877216,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.3645935058594,
      "epoch": 0.5330772923667787,
      "grad_norm": 0.2138671875,
      "learning_rate": 5.765445880472573e-07,
      "loss": 0.091,
      "num_tokens": 239491647.0,
      "reward": 0.5208333507180214,
      "reward_std": 0.31878695264458656,
      "rewards/accuracy_reward": 0.11458333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 1388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.7396240234375,
      "epoch": 0.5334613538166106,
      "grad_norm": 0.1748046875,
      "learning_rate": 5.759419789573773e-07,
      "loss": 0.0652,
      "num_tokens": 239674294.0,
      "reward": 0.369791679084301,
      "reward_std": 0.11454565264284611,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 1389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1476.1354370117188,
      "epoch": 0.5338454152664426,
      "grad_norm": 0.26171875,
      "learning_rate": 5.753393231876957e-07,
      "loss": 0.1237,
      "num_tokens": 239829427.0,
      "reward": 0.6093750149011612,
      "reward_std": 0.3056614249944687,
      "rewards/accuracy_reward": 0.15624999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 1390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.5000610351562,
      "epoch": 0.5342294767162746,
      "grad_norm": 0.19921875,
      "learning_rate": 5.747366218226266e-07,
      "loss": 0.1422,
      "num_tokens": 240012795.0,
      "reward": 0.416666679084301,
      "reward_std": 0.21228357404470444,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 1391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.291748046875,
      "epoch": 0.5346135381661066,
      "grad_norm": 0.228515625,
      "learning_rate": 5.741338759466665e-07,
      "loss": 0.0534,
      "num_tokens": 240193447.0,
      "reward": 0.408854179084301,
      "reward_std": 0.13667036406695843,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1470.0625305175781,
      "epoch": 0.5349975996159385,
      "grad_norm": 0.2490234375,
      "learning_rate": 5.735310866443917e-07,
      "loss": 0.0478,
      "num_tokens": 240348549.0,
      "reward": 0.6171875149011612,
      "reward_std": 0.2790829986333847,
      "rewards/accuracy_reward": 0.21875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.3854370117188,
      "epoch": 0.5353816610657706,
      "grad_norm": 0.23046875,
      "learning_rate": 5.729282550004568e-07,
      "loss": 0.1567,
      "num_tokens": 240517754.0,
      "reward": 0.440104179084301,
      "reward_std": 0.20108774257823825,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1309.8646240234375,
      "epoch": 0.5357657225156025,
      "grad_norm": 0.26171875,
      "learning_rate": 5.723253820995926e-07,
      "loss": 0.0659,
      "num_tokens": 240657197.0,
      "reward": 0.6484375223517418,
      "reward_std": 0.30599462799727917,
      "rewards/accuracy_reward": 0.1770833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.471354179084301,
      "step": 1395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.46875,
      "epoch": 0.5361497839654344,
      "grad_norm": 0.2109375,
      "learning_rate": 5.717224690266038e-07,
      "loss": 0.1501,
      "num_tokens": 240826106.0,
      "reward": 0.4765625223517418,
      "reward_std": 0.2151479236781597,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.4791870117188,
      "epoch": 0.5365338454152665,
      "grad_norm": 0.1962890625,
      "learning_rate": 5.711195168663678e-07,
      "loss": 0.0926,
      "num_tokens": 240994864.0,
      "reward": 0.4739583507180214,
      "reward_std": 0.24028497748076916,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 1397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.3542175292969,
      "epoch": 0.5369179068650984,
      "grad_norm": 0.259765625,
      "learning_rate": 5.705165267038322e-07,
      "loss": 0.1406,
      "num_tokens": 241156674.0,
      "reward": 0.5937500149011612,
      "reward_std": 0.3255101516842842,
      "rewards/accuracy_reward": 0.16666666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1256.3125457763672,
      "epoch": 0.5373019683149304,
      "grad_norm": 0.3203125,
      "learning_rate": 5.699134996240132e-07,
      "loss": 0.1104,
      "num_tokens": 241291200.0,
      "reward": 0.625,
      "reward_std": 0.3328167535364628,
      "rewards/accuracy_reward": 0.13541667070239782,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4895833432674408,
      "step": 1399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.5417175292969,
      "epoch": 0.5376860297647623,
      "grad_norm": 0.2255859375,
      "learning_rate": 5.693104367119925e-07,
      "loss": 0.0849,
      "num_tokens": 241454996.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.23120970278978348,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531249925494194,
      "step": 1400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.6458435058594,
      "epoch": 0.5380700912145944,
      "grad_norm": 0.208984375,
      "learning_rate": 5.687073390529177e-07,
      "loss": 0.0837,
      "num_tokens": 241628002.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.19652432203292847,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.3021545410156,
      "epoch": 0.5384541526644263,
      "grad_norm": 0.1923828125,
      "learning_rate": 5.681042077319977e-07,
      "loss": 0.0903,
      "num_tokens": 241791535.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.1130132544785738,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.34375,
      "epoch": 0.5388382141142583,
      "grad_norm": 0.232421875,
      "learning_rate": 5.675010438345028e-07,
      "loss": 0.1295,
      "num_tokens": 241972600.0,
      "reward": 0.4661458358168602,
      "reward_std": 0.2300941701978445,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 1403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.0417175292969,
      "epoch": 0.5392222755640903,
      "grad_norm": 0.1962890625,
      "learning_rate": 5.668978484457613e-07,
      "loss": 0.067,
      "num_tokens": 242126908.0,
      "reward": 0.611979179084301,
      "reward_std": 0.19173324620351195,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 1404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.4063110351562,
      "epoch": 0.5396063370139222,
      "grad_norm": 0.197265625,
      "learning_rate": 5.662946226511585e-07,
      "loss": 0.0734,
      "num_tokens": 242309475.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.16597985476255417,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1658.4063110351562,
      "epoch": 0.5399903984637542,
      "grad_norm": 0.21875,
      "learning_rate": 5.656913675361344e-07,
      "loss": 0.0915,
      "num_tokens": 242483834.0,
      "reward": 0.4453125223517418,
      "reward_std": 0.2041638121008873,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1381.0104675292969,
      "epoch": 0.5403744599135861,
      "grad_norm": 0.2255859375,
      "learning_rate": 5.650880841861817e-07,
      "loss": 0.1508,
      "num_tokens": 242632803.0,
      "reward": 0.8020833358168602,
      "reward_std": 0.32550282403826714,
      "rewards/accuracy_reward": 0.28125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5208333358168602,
      "step": 1407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.6563110351562,
      "epoch": 0.5407585213634182,
      "grad_norm": 0.2451171875,
      "learning_rate": 5.64484773686844e-07,
      "loss": 0.1367,
      "num_tokens": 242803866.0,
      "reward": 0.408854179084301,
      "reward_std": 0.17150608822703362,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1412.2812805175781,
      "epoch": 0.5411425828132501,
      "grad_norm": 0.21875,
      "learning_rate": 5.638814371237136e-07,
      "loss": 0.179,
      "num_tokens": 242953797.0,
      "reward": 0.5625000223517418,
      "reward_std": 0.27517398446798325,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.479166679084301,
      "step": 1409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.6458435058594,
      "epoch": 0.5415266442630821,
      "grad_norm": 0.232421875,
      "learning_rate": 5.632780755824295e-07,
      "loss": 0.0806,
      "num_tokens": 243112155.0,
      "reward": 0.541666679084301,
      "reward_std": 0.31946030631661415,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.3437805175781,
      "epoch": 0.5419107057129141,
      "grad_norm": 0.19140625,
      "learning_rate": 5.626746901486765e-07,
      "loss": 0.105,
      "num_tokens": 243290860.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.23569084703922272,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583432674408,
      "step": 1411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.6979370117188,
      "epoch": 0.542294767162746,
      "grad_norm": 0.365234375,
      "learning_rate": 5.620712819081814e-07,
      "loss": 0.1314,
      "num_tokens": 243454615.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.1933986060321331,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 1412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.0313110351562,
      "epoch": 0.542678828612578,
      "grad_norm": 0.2080078125,
      "learning_rate": 5.614678519467125e-07,
      "loss": 0.0721,
      "num_tokens": 243623586.0,
      "reward": 0.5520833507180214,
      "reward_std": 0.1891636736690998,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 1413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1411.5625305175781,
      "epoch": 0.5430628900624099,
      "grad_norm": 0.2314453125,
      "learning_rate": 5.608644013500773e-07,
      "loss": 0.1455,
      "num_tokens": 243773784.0,
      "reward": 0.5234375223517418,
      "reward_std": 0.1945267915725708,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 1414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1417.2917175292969,
      "epoch": 0.543446951512242,
      "grad_norm": 0.27734375,
      "learning_rate": 5.602609312041199e-07,
      "loss": 0.1294,
      "num_tokens": 243925284.0,
      "reward": 0.6093750298023224,
      "reward_std": 0.2875711973756552,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 1415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.0313110351562,
      "epoch": 0.5438310129620739,
      "grad_norm": 0.1806640625,
      "learning_rate": 5.596574425947203e-07,
      "loss": 0.1215,
      "num_tokens": 244110751.0,
      "reward": 0.3984375149011612,
      "reward_std": 0.19522362388670444,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 1416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.072998046875,
      "epoch": 0.5442150744119059,
      "grad_norm": 0.22265625,
      "learning_rate": 5.590539366077911e-07,
      "loss": 0.1307,
      "num_tokens": 244284350.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.2044506436213851,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 1417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.6354370117188,
      "epoch": 0.5445991358617379,
      "grad_norm": 0.279296875,
      "learning_rate": 5.584504143292765e-07,
      "loss": 0.113,
      "num_tokens": 244452331.0,
      "reward": 0.4843750074505806,
      "reward_std": 0.24690554291009903,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.5521240234375,
      "epoch": 0.5449831973115699,
      "grad_norm": 0.1767578125,
      "learning_rate": 5.578468768451501e-07,
      "loss": 0.0378,
      "num_tokens": 244630296.0,
      "reward": 0.4322916865348816,
      "reward_std": 0.18176649045199156,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.0312805175781,
      "epoch": 0.5453672587614018,
      "grad_norm": 0.2373046875,
      "learning_rate": 5.572433252414127e-07,
      "loss": 0.102,
      "num_tokens": 244793251.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.2602669633924961,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1395.7708740234375,
      "epoch": 0.5457513202112338,
      "grad_norm": 0.2470703125,
      "learning_rate": 5.566397606040903e-07,
      "loss": 0.0899,
      "num_tokens": 244940597.0,
      "reward": 0.5703125149011612,
      "reward_std": 0.18728622421622276,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625074505806,
      "step": 1421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.3021545410156,
      "epoch": 0.5461353816610658,
      "grad_norm": 0.2060546875,
      "learning_rate": 5.560361840192325e-07,
      "loss": 0.1008,
      "num_tokens": 245102618.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.27817194163799286,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 1422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.1562805175781,
      "epoch": 0.5465194431108977,
      "grad_norm": 0.2333984375,
      "learning_rate": 5.554325965729107e-07,
      "loss": 0.0263,
      "num_tokens": 245272633.0,
      "reward": 0.440104179084301,
      "reward_std": 0.17008555214852095,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.2291870117188,
      "epoch": 0.5469035045607297,
      "grad_norm": 0.25,
      "learning_rate": 5.548289993512158e-07,
      "loss": 0.0794,
      "num_tokens": 245446887.0,
      "reward": 0.4895833432674408,
      "reward_std": 0.22196505032479763,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.8541870117188,
      "epoch": 0.5472875660105617,
      "grad_norm": 0.2158203125,
      "learning_rate": 5.542253934402555e-07,
      "loss": 0.1171,
      "num_tokens": 245608337.0,
      "reward": 0.6354167014360428,
      "reward_std": 0.2966262400150299,
      "rewards/accuracy_reward": 0.20833332743495703,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.6042175292969,
      "epoch": 0.5476716274603937,
      "grad_norm": 0.19921875,
      "learning_rate": 5.536217799261542e-07,
      "loss": 0.1104,
      "num_tokens": 245792115.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.14563214406371117,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1525.0208587646484,
      "epoch": 0.5480556889102256,
      "grad_norm": 0.244140625,
      "learning_rate": 5.530181598950493e-07,
      "loss": 0.0689,
      "num_tokens": 245956389.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.21444932371377945,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1540.0000305175781,
      "epoch": 0.5484397503600577,
      "grad_norm": 0.2158203125,
      "learning_rate": 5.524145344330903e-07,
      "loss": 0.0758,
      "num_tokens": 246117901.0,
      "reward": 0.5625000298023224,
      "reward_std": 0.25805341452360153,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 1428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.6875610351562,
      "epoch": 0.5488238118098896,
      "grad_norm": 0.1943359375,
      "learning_rate": 5.51810904626436e-07,
      "loss": 0.0858,
      "num_tokens": 246291215.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.1543102371506393,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.7708435058594,
      "epoch": 0.5492078732597215,
      "grad_norm": 0.2470703125,
      "learning_rate": 5.512072715612539e-07,
      "loss": 0.183,
      "num_tokens": 246456361.0,
      "reward": 0.520833358168602,
      "reward_std": 0.2680138051509857,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 1430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.0104370117188,
      "epoch": 0.5495919347095535,
      "grad_norm": 0.17578125,
      "learning_rate": 5.506036363237163e-07,
      "loss": 0.1231,
      "num_tokens": 246616466.0,
      "reward": 0.4505208507180214,
      "reward_std": 0.14472223911434412,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.3646240234375,
      "epoch": 0.5499759961593855,
      "grad_norm": 0.2119140625,
      "learning_rate": 5.5e-07,
      "loss": 0.0947,
      "num_tokens": 246800893.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.1648496901616454,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.5937805175781,
      "epoch": 0.5503600576092175,
      "grad_norm": 0.2099609375,
      "learning_rate": 5.493963636762837e-07,
      "loss": 0.1032,
      "num_tokens": 246983446.0,
      "reward": 0.518229179084301,
      "reward_std": 0.32439757138490677,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 1433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.8021240234375,
      "epoch": 0.5507441190590494,
      "grad_norm": 0.2236328125,
      "learning_rate": 5.487927284387462e-07,
      "loss": 0.1402,
      "num_tokens": 247159635.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.2109045758843422,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.7917175292969,
      "epoch": 0.5511281805088815,
      "grad_norm": 0.2001953125,
      "learning_rate": 5.481890953735638e-07,
      "loss": 0.1339,
      "num_tokens": 247347103.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.1780599281191826,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.0000610351562,
      "epoch": 0.5515122419587134,
      "grad_norm": 0.224609375,
      "learning_rate": 5.475854655669098e-07,
      "loss": 0.1268,
      "num_tokens": 247521319.0,
      "reward": 0.4036458358168602,
      "reward_std": 0.12785421684384346,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.2292175292969,
      "epoch": 0.5518963034085453,
      "grad_norm": 0.216796875,
      "learning_rate": 5.469818401049507e-07,
      "loss": 0.0807,
      "num_tokens": 247706397.0,
      "reward": 0.4140625223517418,
      "reward_std": 0.15298773627728224,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 1437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1354.6667175292969,
      "epoch": 0.5522803648583774,
      "grad_norm": 0.271484375,
      "learning_rate": 5.46378220073846e-07,
      "loss": 0.0961,
      "num_tokens": 247849749.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.19739755801856518,
      "rewards/accuracy_reward": 0.078125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1328.4167175292969,
      "epoch": 0.5526644263082093,
      "grad_norm": 0.34765625,
      "learning_rate": 5.457746065597445e-07,
      "loss": 0.1119,
      "num_tokens": 247994685.0,
      "reward": 0.5000000298023224,
      "reward_std": 0.20519467070698738,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4895833283662796,
      "step": 1439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.6666870117188,
      "epoch": 0.5530484877580413,
      "grad_norm": 0.23046875,
      "learning_rate": 5.451710006487844e-07,
      "loss": 0.1197,
      "num_tokens": 248155005.0,
      "reward": 0.5208333507180214,
      "reward_std": 0.26480305567383766,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166567325592,
      "step": 1440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.2604675292969,
      "epoch": 0.5534325492078732,
      "grad_norm": 0.203125,
      "learning_rate": 5.445674034270893e-07,
      "loss": 0.1171,
      "num_tokens": 248341526.0,
      "reward": 0.3932291716337204,
      "reward_std": 0.1458445768803358,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 1441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.2291870117188,
      "epoch": 0.5538166106577053,
      "grad_norm": 0.24609375,
      "learning_rate": 5.439638159807676e-07,
      "loss": 0.0776,
      "num_tokens": 248503132.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.26705484092235565,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458283662796,
      "step": 1442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.5521545410156,
      "epoch": 0.5542006721075372,
      "grad_norm": 6.5,
      "learning_rate": 5.433602393959098e-07,
      "loss": 0.0487,
      "num_tokens": 248667737.0,
      "reward": 0.3984375074505806,
      "reward_std": 0.16219702269881964,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 1443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.1458740234375,
      "epoch": 0.5545847335573691,
      "grad_norm": 0.296875,
      "learning_rate": 5.427566747585874e-07,
      "loss": 0.1011,
      "num_tokens": 248831215.0,
      "reward": 0.5885416865348816,
      "reward_std": 0.2877453751862049,
      "rewards/accuracy_reward": 0.1562500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.7084045410156,
      "epoch": 0.5549687950072012,
      "grad_norm": 0.203125,
      "learning_rate": 5.421531231548499e-07,
      "loss": 0.115,
      "num_tokens": 249013043.0,
      "reward": 0.5286458432674408,
      "reward_std": 0.3213389292359352,
      "rewards/accuracy_reward": 0.13541666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.5938110351562,
      "epoch": 0.5553528564570331,
      "grad_norm": 0.2021484375,
      "learning_rate": 5.415495856707235e-07,
      "loss": 0.1375,
      "num_tokens": 249178436.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.20918704569339752,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 1446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1845.9583740234375,
      "epoch": 0.5557369179068651,
      "grad_norm": 0.1669921875,
      "learning_rate": 5.40946063392209e-07,
      "loss": 0.0778,
      "num_tokens": 249373936.0,
      "reward": 0.3411458432674408,
      "reward_std": 0.14540416141971946,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3098958358168602,
      "step": 1447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.5209045410156,
      "epoch": 0.556120979356697,
      "grad_norm": 0.62109375,
      "learning_rate": 5.403425574052798e-07,
      "loss": -0.133,
      "num_tokens": 249545274.0,
      "reward": 0.3750000149011612,
      "reward_std": 0.16408849507570267,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 1448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.1146240234375,
      "epoch": 0.5565050408065291,
      "grad_norm": 0.201171875,
      "learning_rate": 5.397390687958801e-07,
      "loss": 0.1037,
      "num_tokens": 249727509.0,
      "reward": 0.5104166716337204,
      "reward_std": 0.20949982479214668,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.4688110351562,
      "epoch": 0.556889102256361,
      "grad_norm": 0.2265625,
      "learning_rate": 5.391355986499228e-07,
      "loss": 0.0597,
      "num_tokens": 249889874.0,
      "reward": 0.580729179084301,
      "reward_std": 0.3345729373395443,
      "rewards/accuracy_reward": 0.14583333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1508.3125610351562,
      "epoch": 0.557273163706193,
      "grad_norm": 0.2373046875,
      "learning_rate": 5.385321480532875e-07,
      "loss": 0.1338,
      "num_tokens": 250051488.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.22890101745724678,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583358168602,
      "step": 1451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.5104675292969,
      "epoch": 0.557657225156025,
      "grad_norm": 0.23828125,
      "learning_rate": 5.379287180918185e-07,
      "loss": 0.1336,
      "num_tokens": 250213569.0,
      "reward": 0.5625000074505806,
      "reward_std": 0.24832214415073395,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1477.0104370117188,
      "epoch": 0.558041286605857,
      "grad_norm": 0.234375,
      "learning_rate": 5.373253098513235e-07,
      "loss": 0.065,
      "num_tokens": 250368258.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.15131642948836088,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.0625610351562,
      "epoch": 0.5584253480556889,
      "grad_norm": 0.2021484375,
      "learning_rate": 5.367219244175704e-07,
      "loss": 0.0835,
      "num_tokens": 250543440.0,
      "reward": 0.5729166865348816,
      "reward_std": 0.25147397071123123,
      "rewards/accuracy_reward": 0.16666666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.7083740234375,
      "epoch": 0.5588094095055209,
      "grad_norm": 0.1826171875,
      "learning_rate": 5.361185628762865e-07,
      "loss": 0.07,
      "num_tokens": 250723836.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.139083047863096,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1837.8021240234375,
      "epoch": 0.5591934709553529,
      "grad_norm": 0.185546875,
      "learning_rate": 5.355152263131561e-07,
      "loss": 0.0833,
      "num_tokens": 250915409.0,
      "reward": 0.3776041865348816,
      "reward_std": 0.14570233970880508,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041567325592,
      "step": 1456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.8750305175781,
      "epoch": 0.5595775324051848,
      "grad_norm": 0.1591796875,
      "learning_rate": 5.349119158138183e-07,
      "loss": 0.0543,
      "num_tokens": 251092653.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.21275050286203623,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.8542175292969,
      "epoch": 0.5599615938550168,
      "grad_norm": 0.20703125,
      "learning_rate": 5.343086324638656e-07,
      "loss": 0.0433,
      "num_tokens": 251271975.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.17429978400468826,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 1458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.6354675292969,
      "epoch": 0.5603456553048488,
      "grad_norm": 0.20703125,
      "learning_rate": 5.337053773488416e-07,
      "loss": 0.0702,
      "num_tokens": 251440028.0,
      "reward": 0.552083358168602,
      "reward_std": 0.19999536126852036,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.0000610351562,
      "epoch": 0.5607297167546808,
      "grad_norm": 0.2080078125,
      "learning_rate": 5.331021515542387e-07,
      "loss": 0.142,
      "num_tokens": 251620716.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.17016203980892897,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.0313110351562,
      "epoch": 0.5611137782045127,
      "grad_norm": 0.228515625,
      "learning_rate": 5.324989561654973e-07,
      "loss": 0.1128,
      "num_tokens": 251800655.0,
      "reward": 0.424479179084301,
      "reward_std": 0.19071370735764503,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.7500610351562,
      "epoch": 0.5614978396543447,
      "grad_norm": 0.25,
      "learning_rate": 5.318957922680023e-07,
      "loss": 0.0768,
      "num_tokens": 251968271.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.26525118947029114,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.9895935058594,
      "epoch": 0.5618819011041767,
      "grad_norm": 0.2265625,
      "learning_rate": 5.312926609470822e-07,
      "loss": 0.0955,
      "num_tokens": 252136830.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.15414446126669645,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666567325592,
      "step": 1463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1823.8438110351562,
      "epoch": 0.5622659625540086,
      "grad_norm": 0.1728515625,
      "learning_rate": 5.306895632880074e-07,
      "loss": 0.0864,
      "num_tokens": 252324951.0,
      "reward": 0.3880208358168602,
      "reward_std": 0.16217366605997086,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375074505806,
      "step": 1464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.3229370117188,
      "epoch": 0.5626500240038406,
      "grad_norm": 0.201171875,
      "learning_rate": 5.300865003759869e-07,
      "loss": 0.0891,
      "num_tokens": 252502694.0,
      "reward": 0.598958358168602,
      "reward_std": 0.2957101874053478,
      "rewards/accuracy_reward": 0.1875000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.2500305175781,
      "epoch": 0.5630340854536726,
      "grad_norm": 0.2109375,
      "learning_rate": 5.294834732961677e-07,
      "loss": 0.1172,
      "num_tokens": 252677710.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.19187234714627266,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 1466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.3333740234375,
      "epoch": 0.5634181469035046,
      "grad_norm": 0.1669921875,
      "learning_rate": 5.288804831336321e-07,
      "loss": 0.0927,
      "num_tokens": 252863454.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.16113201715052128,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 1467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.1042175292969,
      "epoch": 0.5638022083533365,
      "grad_norm": 0.23046875,
      "learning_rate": 5.282775309733962e-07,
      "loss": 0.1375,
      "num_tokens": 253036208.0,
      "reward": 0.4166666716337204,
      "reward_std": 0.13431640528142452,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.8437805175781,
      "epoch": 0.5641862698031685,
      "grad_norm": 0.2109375,
      "learning_rate": 5.276746179004074e-07,
      "loss": 0.0873,
      "num_tokens": 253207321.0,
      "reward": 0.4661458358168602,
      "reward_std": 0.2317754067480564,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1604.8125610351562,
      "epoch": 0.5645703312530005,
      "grad_norm": 0.2177734375,
      "learning_rate": 5.270717449995432e-07,
      "loss": 0.0895,
      "num_tokens": 253378703.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.14930124394595623,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.2708435058594,
      "epoch": 0.5649543927028324,
      "grad_norm": 0.1943359375,
      "learning_rate": 5.264689133556084e-07,
      "loss": 0.1207,
      "num_tokens": 253559481.0,
      "reward": 0.4505208507180214,
      "reward_std": 0.23817376047372818,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 1471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.9062805175781,
      "epoch": 0.5653384541526644,
      "grad_norm": 0.259765625,
      "learning_rate": 5.258661240533336e-07,
      "loss": 0.0827,
      "num_tokens": 253724336.0,
      "reward": 0.5572916716337204,
      "reward_std": 0.27987657859921455,
      "rewards/accuracy_reward": 0.12500000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916567325592,
      "step": 1472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1864.6354675292969,
      "epoch": 0.5657225156024964,
      "grad_norm": 0.171875,
      "learning_rate": 5.252633781773736e-07,
      "loss": 0.0715,
      "num_tokens": 253917349.0,
      "reward": 0.3281250149011612,
      "reward_std": 0.08335855510085821,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.328125,
      "step": 1473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1458.6458740234375,
      "epoch": 0.5661065770523284,
      "grad_norm": 0.21875,
      "learning_rate": 5.246606768123045e-07,
      "loss": 0.0837,
      "num_tokens": 254072107.0,
      "reward": 0.763020858168602,
      "reward_std": 0.3510252833366394,
      "rewards/accuracy_reward": 0.3125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 1474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1610.5000610351562,
      "epoch": 0.5664906385021603,
      "grad_norm": 0.267578125,
      "learning_rate": 5.240580210426228e-07,
      "loss": 0.111,
      "num_tokens": 254242827.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.24814881943166256,
      "rewards/accuracy_reward": 0.0572916679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.5208740234375,
      "epoch": 0.5668746999519924,
      "grad_norm": 0.17578125,
      "learning_rate": 5.234554119527426e-07,
      "loss": 0.0622,
      "num_tokens": 254428221.0,
      "reward": 0.3958333358168602,
      "reward_std": 0.12254573497921228,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 1476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1528.7396545410156,
      "epoch": 0.5672587614018243,
      "grad_norm": 0.25390625,
      "learning_rate": 5.228528506269945e-07,
      "loss": 0.0974,
      "num_tokens": 254587828.0,
      "reward": 0.5781250223517418,
      "reward_std": 0.30654504150152206,
      "rewards/accuracy_reward": 0.13541666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1392.8437805175781,
      "epoch": 0.5676428228516562,
      "grad_norm": 0.3046875,
      "learning_rate": 5.222503381496221e-07,
      "loss": 0.0936,
      "num_tokens": 254736485.0,
      "reward": 0.5937500223517418,
      "reward_std": 0.26820098608732224,
      "rewards/accuracy_reward": 0.16145833395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 1478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.9688110351562,
      "epoch": 0.5680268843014883,
      "grad_norm": 0.181640625,
      "learning_rate": 5.216478756047828e-07,
      "loss": 0.0575,
      "num_tokens": 254922690.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.20252296701073647,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291567325592,
      "step": 1479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.8333740234375,
      "epoch": 0.5684109457513202,
      "grad_norm": 0.2197265625,
      "learning_rate": 5.210454640765424e-07,
      "loss": 0.1008,
      "num_tokens": 255107066.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.2083493247628212,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1866.1354675292969,
      "epoch": 0.5687950072011522,
      "grad_norm": 0.2216796875,
      "learning_rate": 5.20443104648876e-07,
      "loss": 0.0846,
      "num_tokens": 255302879.0,
      "reward": 0.3593750074505806,
      "reward_std": 0.14019516296684742,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 1481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.0521545410156,
      "epoch": 0.5691790686509841,
      "grad_norm": 0.1845703125,
      "learning_rate": 5.198407984056647e-07,
      "loss": 0.1037,
      "num_tokens": 255485988.0,
      "reward": 0.471354179084301,
      "reward_std": 0.19474192522466183,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 1482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1844.3854370117188,
      "epoch": 0.5695631301008162,
      "grad_norm": 0.193359375,
      "learning_rate": 5.192385464306934e-07,
      "loss": 0.0646,
      "num_tokens": 255677353.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.18545327335596085,
      "rewards/accuracy_reward": 0.1249999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 1483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1380.8750305175781,
      "epoch": 0.5699471915506481,
      "grad_norm": 0.33984375,
      "learning_rate": 5.186363498076502e-07,
      "loss": 0.0376,
      "num_tokens": 255823477.0,
      "reward": 0.5520833432674408,
      "reward_std": 0.19889827026054263,
      "rewards/accuracy_reward": 0.12500000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.1562805175781,
      "epoch": 0.57033125300048,
      "grad_norm": 0.2001953125,
      "learning_rate": 5.180342096201229e-07,
      "loss": 0.0977,
      "num_tokens": 255981572.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.16597411781549454,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 1485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1521.5521240234375,
      "epoch": 0.5707153144503121,
      "grad_norm": 0.232421875,
      "learning_rate": 5.17432126951598e-07,
      "loss": 0.1781,
      "num_tokens": 256145753.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.14383646845817566,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1434.0729370117188,
      "epoch": 0.571099375900144,
      "grad_norm": 0.232421875,
      "learning_rate": 5.168301028854588e-07,
      "loss": 0.1367,
      "num_tokens": 256297616.0,
      "reward": 0.6744791865348816,
      "reward_std": 0.29840903729200363,
      "rewards/accuracy_reward": 0.19791666232049465,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 1487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.4479370117188,
      "epoch": 0.571483437349976,
      "grad_norm": 0.25,
      "learning_rate": 5.162281385049827e-07,
      "loss": 0.0859,
      "num_tokens": 256457955.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.22221830114722252,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4869791641831398,
      "step": 1488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1452.4375305175781,
      "epoch": 0.5718674987998079,
      "grad_norm": 0.275390625,
      "learning_rate": 5.156262348933399e-07,
      "loss": 0.1215,
      "num_tokens": 256610605.0,
      "reward": 0.5338541865348816,
      "reward_std": 0.2285987287759781,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 1489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.8750610351562,
      "epoch": 0.57225156024964,
      "grad_norm": 0.26171875,
      "learning_rate": 5.15024393133591e-07,
      "loss": 0.0929,
      "num_tokens": 256767305.0,
      "reward": 0.5000000223517418,
      "reward_std": 0.2384241484105587,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 1490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.4896087646484,
      "epoch": 0.5726356216994719,
      "grad_norm": 0.1953125,
      "learning_rate": 5.14422614308686e-07,
      "loss": 0.0758,
      "num_tokens": 256943776.0,
      "reward": 0.5859375223517418,
      "reward_std": 0.23412062600255013,
      "rewards/accuracy_reward": 0.20833333861082792,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 1491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.1563110351562,
      "epoch": 0.5730196831493038,
      "grad_norm": 0.232421875,
      "learning_rate": 5.138208995014608e-07,
      "loss": 0.1125,
      "num_tokens": 257102063.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.1543233636766672,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.6146240234375,
      "epoch": 0.5734037445991359,
      "grad_norm": 0.283203125,
      "learning_rate": 5.132192497946364e-07,
      "loss": 0.1487,
      "num_tokens": 257269562.0,
      "reward": 0.5677083432674408,
      "reward_std": 0.22334402799606323,
      "rewards/accuracy_reward": 0.16666666232049465,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.0833740234375,
      "epoch": 0.5737878060489678,
      "grad_norm": 0.251953125,
      "learning_rate": 5.126176662708174e-07,
      "loss": 0.0821,
      "num_tokens": 257435066.0,
      "reward": 0.5468750149011612,
      "reward_std": 0.2963280789554119,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.0104675292969,
      "epoch": 0.5741718674987998,
      "grad_norm": 0.134765625,
      "learning_rate": 5.120161500124879e-07,
      "loss": 0.0524,
      "num_tokens": 257600539.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.0663489019498229,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 1495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.5104675292969,
      "epoch": 0.5745559289486318,
      "grad_norm": 0.234375,
      "learning_rate": 5.114147021020124e-07,
      "loss": 0.1278,
      "num_tokens": 257768788.0,
      "reward": 0.4765625,
      "reward_std": 0.18573805317282677,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.7917175292969,
      "epoch": 0.5749399903984638,
      "grad_norm": 0.2109375,
      "learning_rate": 5.108133236216309e-07,
      "loss": 0.048,
      "num_tokens": 257946952.0,
      "reward": 0.4505208507180214,
      "reward_std": 0.1831425242125988,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.9584045410156,
      "epoch": 0.5753240518482957,
      "grad_norm": 0.1865234375,
      "learning_rate": 5.102120156534603e-07,
      "loss": 0.1363,
      "num_tokens": 258115860.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.2756210118532181,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.1146240234375,
      "epoch": 0.5757081132981277,
      "grad_norm": 0.2216796875,
      "learning_rate": 5.096107792794887e-07,
      "loss": 0.0639,
      "num_tokens": 258284791.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.25990596786141396,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.8229675292969,
      "epoch": 0.5760921747479597,
      "grad_norm": 0.2294921875,
      "learning_rate": 5.090096155815772e-07,
      "loss": 0.1247,
      "num_tokens": 258451926.0,
      "reward": 0.4895833507180214,
      "reward_std": 0.18537478148937225,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.2812805175781,
      "epoch": 0.5764762361977916,
      "grad_norm": 0.255859375,
      "learning_rate": 5.084085256414543e-07,
      "loss": 0.0764,
      "num_tokens": 258621809.0,
      "reward": 0.463541679084301,
      "reward_std": 0.23268011957406998,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583432674408,
      "step": 1501
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.7708740234375,
      "epoch": 0.5768602976476236,
      "grad_norm": 0.1806640625,
      "learning_rate": 5.078075105407177e-07,
      "loss": 0.1021,
      "num_tokens": 258801843.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.1941059436649084,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.3750305175781,
      "epoch": 0.5772443590974556,
      "grad_norm": 0.1982421875,
      "learning_rate": 5.072065713608285e-07,
      "loss": 0.0154,
      "num_tokens": 258972263.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.21183743327856064,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1503
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1497.5625610351562,
      "epoch": 0.5776284205472876,
      "grad_norm": 0.265625,
      "learning_rate": 5.066057091831129e-07,
      "loss": 0.0293,
      "num_tokens": 259132205.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.1321097845211625,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1504
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.8438110351562,
      "epoch": 0.5780124819971195,
      "grad_norm": 0.2421875,
      "learning_rate": 5.060049250887573e-07,
      "loss": 0.1184,
      "num_tokens": 259305310.0,
      "reward": 0.416666679084301,
      "reward_std": 0.15451055578887463,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 1505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.0938110351562,
      "epoch": 0.5783965434469515,
      "grad_norm": 0.189453125,
      "learning_rate": 5.054042201588082e-07,
      "loss": 0.0956,
      "num_tokens": 259489151.0,
      "reward": 0.450520858168602,
      "reward_std": 0.1796140819787979,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.6979370117188,
      "epoch": 0.5787806048967835,
      "grad_norm": 0.2021484375,
      "learning_rate": 5.048035954741695e-07,
      "loss": 0.1113,
      "num_tokens": 259655826.0,
      "reward": 0.544270858168602,
      "reward_std": 0.3086201511323452,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1507
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1504.5313110351562,
      "epoch": 0.5791646663466155,
      "grad_norm": 0.2451171875,
      "learning_rate": 5.042030521156007e-07,
      "loss": 0.1563,
      "num_tokens": 259813085.0,
      "reward": 0.4947916716337204,
      "reward_std": 0.24612721800804138,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.0208435058594,
      "epoch": 0.5795487277964474,
      "grad_norm": 0.2158203125,
      "learning_rate": 5.036025911637154e-07,
      "loss": 0.1118,
      "num_tokens": 259984287.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.18158269487321377,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 1509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.6875305175781,
      "epoch": 0.5799327892462794,
      "grad_norm": 0.26953125,
      "learning_rate": 5.030022136989779e-07,
      "loss": 0.1225,
      "num_tokens": 260151273.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.1803081687539816,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1510
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.8541870117188,
      "epoch": 0.5803168506961114,
      "grad_norm": 0.2080078125,
      "learning_rate": 5.024019208017034e-07,
      "loss": 0.0768,
      "num_tokens": 260331291.0,
      "reward": 0.393229179084301,
      "reward_std": 0.11509960424154997,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.5834045410156,
      "epoch": 0.5807009121459433,
      "grad_norm": 0.2060546875,
      "learning_rate": 5.01801713552054e-07,
      "loss": 0.0889,
      "num_tokens": 260502411.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.21274417638778687,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1512
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.3333740234375,
      "epoch": 0.5810849735957754,
      "grad_norm": 0.208984375,
      "learning_rate": 5.012015930300386e-07,
      "loss": 0.1153,
      "num_tokens": 260663987.0,
      "reward": 0.4739583358168602,
      "reward_std": 0.18183445325121284,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.9791870117188,
      "epoch": 0.5814690350456073,
      "grad_norm": 0.189453125,
      "learning_rate": 5.006015603155089e-07,
      "loss": 0.1201,
      "num_tokens": 260845545.0,
      "reward": 0.567708358168602,
      "reward_std": 0.32093360647559166,
      "rewards/accuracy_reward": 0.16666667256504297,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 1514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.2917175292969,
      "epoch": 0.5818530964954393,
      "grad_norm": 0.2216796875,
      "learning_rate": 5.000016164881599e-07,
      "loss": 0.1285,
      "num_tokens": 261018325.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.17860830575227737,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.7083740234375,
      "epoch": 0.5822371579452712,
      "grad_norm": 0.2373046875,
      "learning_rate": 4.994017626275255e-07,
      "loss": 0.1338,
      "num_tokens": 261201457.0,
      "reward": 0.432291679084301,
      "reward_std": 0.19056298583745956,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1516
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1558.7708740234375,
      "epoch": 0.5826212193951033,
      "grad_norm": 0.2119140625,
      "learning_rate": 4.988019998129788e-07,
      "loss": 0.1129,
      "num_tokens": 261365483.0,
      "reward": 0.4921875223517418,
      "reward_std": 0.21031064353883266,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1517
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.8854675292969,
      "epoch": 0.5830052808449352,
      "grad_norm": 0.224609375,
      "learning_rate": 4.982023291237277e-07,
      "loss": 0.0822,
      "num_tokens": 261555024.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.14068455807864666,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3749999925494194,
      "step": 1518
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.4167175292969,
      "epoch": 0.5833893422947671,
      "grad_norm": 0.271484375,
      "learning_rate": 4.97602751638816e-07,
      "loss": 0.1698,
      "num_tokens": 261714992.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.2315070517361164,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583432674408,
      "step": 1519
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.8646240234375,
      "epoch": 0.5837734037445992,
      "grad_norm": 0.228515625,
      "learning_rate": 4.970032684371181e-07,
      "loss": 0.1014,
      "num_tokens": 261879715.0,
      "reward": 0.5520833432674408,
      "reward_std": 0.3443222790956497,
      "rewards/accuracy_reward": 0.12500000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1520
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1678.2291870117188,
      "epoch": 0.5841574651944311,
      "grad_norm": 0.2099609375,
      "learning_rate": 4.964038805973398e-07,
      "loss": 0.0802,
      "num_tokens": 262055121.0,
      "reward": 0.471354179084301,
      "reward_std": 0.17500471509993076,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 1521
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1877.5417175292969,
      "epoch": 0.5845415266442631,
      "grad_norm": 0.1669921875,
      "learning_rate": 4.958045891980154e-07,
      "loss": 0.0895,
      "num_tokens": 262249453.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.20339185697957873,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 1522
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.7187805175781,
      "epoch": 0.584925588094095,
      "grad_norm": 0.181640625,
      "learning_rate": 4.952053953175051e-07,
      "loss": 0.1041,
      "num_tokens": 262424618.0,
      "reward": 0.5677083432674408,
      "reward_std": 0.2856782227754593,
      "rewards/accuracy_reward": 0.16666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 1523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.5417175292969,
      "epoch": 0.5853096495439271,
      "grad_norm": 0.166015625,
      "learning_rate": 4.946063000339942e-07,
      "loss": 0.0612,
      "num_tokens": 262611894.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.15393904922530055,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 1524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1329.0312957763672,
      "epoch": 0.585693710993759,
      "grad_norm": 0.2138671875,
      "learning_rate": 4.940073044254894e-07,
      "loss": 0.0608,
      "num_tokens": 262752185.0,
      "reward": 0.6250000149011612,
      "reward_std": 0.259533129632473,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4895833358168602,
      "step": 1525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.8020935058594,
      "epoch": 0.5860777724435909,
      "grad_norm": 0.18359375,
      "learning_rate": 4.934084095698197e-07,
      "loss": 0.0742,
      "num_tokens": 262938686.0,
      "reward": 0.5078125298023224,
      "reward_std": 0.22302203625440598,
      "rewards/accuracy_reward": 0.1354166716337204,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 1526
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.4896240234375,
      "epoch": 0.586461833893423,
      "grad_norm": 0.208984375,
      "learning_rate": 4.928096165446316e-07,
      "loss": 0.1192,
      "num_tokens": 263125373.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.19945478811860085,
      "rewards/accuracy_reward": 0.03645833395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1527
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1492.5937805175781,
      "epoch": 0.5868458953432549,
      "grad_norm": 0.330078125,
      "learning_rate": 4.922109264273891e-07,
      "loss": 0.1368,
      "num_tokens": 263281142.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.1438254490494728,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1528
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.7917175292969,
      "epoch": 0.5872299567930869,
      "grad_norm": 0.1669921875,
      "learning_rate": 4.9161234029537e-07,
      "loss": 0.0504,
      "num_tokens": 263467226.0,
      "reward": 0.447916679084301,
      "reward_std": 0.16279560280963778,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.34375,
      "step": 1529
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.9792175292969,
      "epoch": 0.5876140182429188,
      "grad_norm": 0.2021484375,
      "learning_rate": 4.910138592256665e-07,
      "loss": 0.1305,
      "num_tokens": 263645176.0,
      "reward": 0.440104179084301,
      "reward_std": 0.19598350953310728,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 1530
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.0833740234375,
      "epoch": 0.5879980796927509,
      "grad_norm": 0.21484375,
      "learning_rate": 4.904154842951803e-07,
      "loss": 0.1224,
      "num_tokens": 263831864.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.22747502103447914,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333432674408,
      "step": 1531
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.2188110351562,
      "epoch": 0.5883821411425828,
      "grad_norm": 0.205078125,
      "learning_rate": 4.898172165806232e-07,
      "loss": 0.1479,
      "num_tokens": 264010077.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.23850945010781288,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 1532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1750.1562805175781,
      "epoch": 0.5887662025924147,
      "grad_norm": 0.19921875,
      "learning_rate": 4.892190571585133e-07,
      "loss": 0.1074,
      "num_tokens": 264192492.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.19339514896273613,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 1533
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.2500305175781,
      "epoch": 0.5891502640422468,
      "grad_norm": 0.1728515625,
      "learning_rate": 4.886210071051745e-07,
      "loss": 0.0909,
      "num_tokens": 264363628.0,
      "reward": 0.4869791716337204,
      "reward_std": 0.22144754184409976,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1534
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.1666870117188,
      "epoch": 0.5895343254920787,
      "grad_norm": 0.21875,
      "learning_rate": 4.880230674967329e-07,
      "loss": 0.0921,
      "num_tokens": 264541636.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.19902379903942347,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 1535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.8333740234375,
      "epoch": 0.5899183869419107,
      "grad_norm": 0.220703125,
      "learning_rate": 4.874252394091173e-07,
      "loss": 0.0788,
      "num_tokens": 264697268.0,
      "reward": 0.6093750074505806,
      "reward_std": 0.2779088318347931,
      "rewards/accuracy_reward": 0.14583333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416641831398,
      "step": 1536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.2812805175781,
      "epoch": 0.5903024483917427,
      "grad_norm": 0.1865234375,
      "learning_rate": 4.868275239180547e-07,
      "loss": 0.1063,
      "num_tokens": 264890943.0,
      "reward": 0.416666679084301,
      "reward_std": 0.1531474068760872,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 1537
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1404.6250305175781,
      "epoch": 0.5906865098415747,
      "grad_norm": 0.25,
      "learning_rate": 4.862299220990698e-07,
      "loss": 0.1006,
      "num_tokens": 265039155.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.1692197136580944,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 1538
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.28125,
      "epoch": 0.5910705712914066,
      "grad_norm": 0.2158203125,
      "learning_rate": 4.856324350274831e-07,
      "loss": 0.1208,
      "num_tokens": 265203542.0,
      "reward": 0.5833333432674408,
      "reward_std": 0.31851108744740486,
      "rewards/accuracy_reward": 0.16666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1539
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.6146545410156,
      "epoch": 0.5914546327412386,
      "grad_norm": 0.234375,
      "learning_rate": 4.85035063778408e-07,
      "loss": 0.0839,
      "num_tokens": 265360321.0,
      "reward": 0.5416666716337204,
      "reward_std": 0.2083098255097866,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833432674408,
      "step": 1540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.9167175292969,
      "epoch": 0.5918386941910706,
      "grad_norm": 0.2109375,
      "learning_rate": 4.8443780942675e-07,
      "loss": 0.0838,
      "num_tokens": 265531305.0,
      "reward": 0.5468750149011612,
      "reward_std": 0.2427374441176653,
      "rewards/accuracy_reward": 0.15624999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1541
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.9583740234375,
      "epoch": 0.5922227556409025,
      "grad_norm": 0.25390625,
      "learning_rate": 4.83840673047204e-07,
      "loss": 0.1829,
      "num_tokens": 265692845.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.23687039129436016,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1542
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.9687805175781,
      "epoch": 0.5926068170907345,
      "grad_norm": 0.21875,
      "learning_rate": 4.832436557142529e-07,
      "loss": 0.0707,
      "num_tokens": 265846402.0,
      "reward": 0.6015625223517418,
      "reward_std": 0.22583849262446165,
      "rewards/accuracy_reward": 0.18749999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1543
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.4792175292969,
      "epoch": 0.5929908785405665,
      "grad_norm": 0.2490234375,
      "learning_rate": 4.826467585021649e-07,
      "loss": 0.1008,
      "num_tokens": 266006864.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.17301607877016068,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1544
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.0104370117188,
      "epoch": 0.5933749399903985,
      "grad_norm": 0.2294921875,
      "learning_rate": 4.820499824849926e-07,
      "loss": 0.051,
      "num_tokens": 266181353.0,
      "reward": 0.471354179084301,
      "reward_std": 0.19404218159615993,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 1545
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.0520935058594,
      "epoch": 0.5937590014402304,
      "grad_norm": 0.1875,
      "learning_rate": 4.8145332873657e-07,
      "loss": 0.0573,
      "num_tokens": 266344590.0,
      "reward": 0.463541679084301,
      "reward_std": 0.18594732321798801,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 1546
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.8958740234375,
      "epoch": 0.5941430628900624,
      "grad_norm": 0.2333984375,
      "learning_rate": 4.808567983305117e-07,
      "loss": 0.1216,
      "num_tokens": 266501028.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.14444144070148468,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625074505806,
      "step": 1547
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1492.2708740234375,
      "epoch": 0.5945271243398944,
      "grad_norm": 0.2490234375,
      "learning_rate": 4.802603923402097e-07,
      "loss": 0.0429,
      "num_tokens": 266657342.0,
      "reward": 0.494791679084301,
      "reward_std": 0.2230701483786106,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416716337204,
      "step": 1548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1849.1354370117188,
      "epoch": 0.5949111857897263,
      "grad_norm": 0.2060546875,
      "learning_rate": 4.796641118388328e-07,
      "loss": 0.08,
      "num_tokens": 266849603.0,
      "reward": 0.3671875074505806,
      "reward_std": 0.15349033754318953,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 1549
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.6562805175781,
      "epoch": 0.5952952472395583,
      "grad_norm": 0.2890625,
      "learning_rate": 4.790679578993232e-07,
      "loss": 0.1609,
      "num_tokens": 267011258.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.27531522884964943,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1550
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.6562805175781,
      "epoch": 0.5956793086893903,
      "grad_norm": 0.205078125,
      "learning_rate": 4.784719315943961e-07,
      "loss": 0.1274,
      "num_tokens": 267172617.0,
      "reward": 0.5208333507180214,
      "reward_std": 0.2329155746847391,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1551
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1588.9583740234375,
      "epoch": 0.5960633701392223,
      "grad_norm": 0.2138671875,
      "learning_rate": 4.778760339965369e-07,
      "loss": 0.1333,
      "num_tokens": 267337157.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.16419445350766182,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1552
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.7083740234375,
      "epoch": 0.5964474315890542,
      "grad_norm": 0.185546875,
      "learning_rate": 4.772802661779988e-07,
      "loss": 0.0809,
      "num_tokens": 267510265.0,
      "reward": 0.471354179084301,
      "reward_std": 0.19990328140556812,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.5208740234375,
      "epoch": 0.5968314930388863,
      "grad_norm": 0.212890625,
      "learning_rate": 4.7668462921080243e-07,
      "loss": 0.0454,
      "num_tokens": 267688219.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.21257181093096733,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 1554
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.3541870117188,
      "epoch": 0.5972155544887182,
      "grad_norm": 0.330078125,
      "learning_rate": 4.760891241667321e-07,
      "loss": -0.0308,
      "num_tokens": 267859981.0,
      "reward": 0.3984375149011612,
      "reward_std": 0.16131441667675972,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 1555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.1041870117188,
      "epoch": 0.5975996159385502,
      "grad_norm": 0.236328125,
      "learning_rate": 4.754937521173352e-07,
      "loss": 0.0882,
      "num_tokens": 268031839.0,
      "reward": 0.4505208507180214,
      "reward_std": 0.19718451984226704,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1556
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.0000305175781,
      "epoch": 0.5979836773883821,
      "grad_norm": 0.234375,
      "learning_rate": 4.748985141339197e-07,
      "loss": 0.1156,
      "num_tokens": 268214711.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.17520455829799175,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.6771240234375,
      "epoch": 0.5983677388382141,
      "grad_norm": 0.2021484375,
      "learning_rate": 4.743034112875525e-07,
      "loss": 0.0947,
      "num_tokens": 268401872.0,
      "reward": 0.3802083432674408,
      "reward_std": 0.1474788561463356,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.5209045410156,
      "epoch": 0.5987518002880461,
      "grad_norm": 0.2197265625,
      "learning_rate": 4.737084446490569e-07,
      "loss": 0.1143,
      "num_tokens": 268584978.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.21048157289624214,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1559
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1726.2500305175781,
      "epoch": 0.599135861737878,
      "grad_norm": 0.2041015625,
      "learning_rate": 4.7311361528901163e-07,
      "loss": 0.1137,
      "num_tokens": 268766402.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.3079546242952347,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 1560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.2083435058594,
      "epoch": 0.5995199231877101,
      "grad_norm": 0.203125,
      "learning_rate": 4.72518924277748e-07,
      "loss": 0.1426,
      "num_tokens": 268946718.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.16152846068143845,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1561
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.8542022705078,
      "epoch": 0.599903984637542,
      "grad_norm": 0.2216796875,
      "learning_rate": 4.719243726853487e-07,
      "loss": 0.0943,
      "num_tokens": 269113048.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.18241295218467712,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1562
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1504.3958740234375,
      "epoch": 0.600288046087374,
      "grad_norm": 0.2431640625,
      "learning_rate": 4.713299615816452e-07,
      "loss": 0.107,
      "num_tokens": 269270470.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.2449122779071331,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 1563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1679.9479370117188,
      "epoch": 0.6006721075372059,
      "grad_norm": 0.232421875,
      "learning_rate": 4.7073569203621657e-07,
      "loss": 0.1003,
      "num_tokens": 269447833.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.16092402674257755,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1564
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.3437805175781,
      "epoch": 0.601056168987038,
      "grad_norm": 0.19921875,
      "learning_rate": 4.7014156511838674e-07,
      "loss": 0.0772,
      "num_tokens": 269617450.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.19666427886113524,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1565
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.1146240234375,
      "epoch": 0.6014402304368699,
      "grad_norm": 0.197265625,
      "learning_rate": 4.6954758189722333e-07,
      "loss": 0.1109,
      "num_tokens": 269797725.0,
      "reward": 0.3984375149011612,
      "reward_std": 0.17282497137784958,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 1566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1338.0208740234375,
      "epoch": 0.6018242918867018,
      "grad_norm": 0.2734375,
      "learning_rate": 4.6895374344153535e-07,
      "loss": 0.1213,
      "num_tokens": 269941535.0,
      "reward": 0.6197916716337204,
      "reward_std": 0.252596240490675,
      "rewards/accuracy_reward": 0.12500000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4947916641831398,
      "step": 1567
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.1562805175781,
      "epoch": 0.6022083533365339,
      "grad_norm": 0.2236328125,
      "learning_rate": 4.6836005081987116e-07,
      "loss": 0.1686,
      "num_tokens": 270117614.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.20510074496269226,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984374925494194,
      "step": 1568
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1691.8437805175781,
      "epoch": 0.6025924147863658,
      "grad_norm": 0.1689453125,
      "learning_rate": 4.6776650510051675e-07,
      "loss": 0.0593,
      "num_tokens": 270296327.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.1158068273216486,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708432674408,
      "step": 1569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.4063110351562,
      "epoch": 0.6029764762361978,
      "grad_norm": 0.2265625,
      "learning_rate": 4.6717310735149386e-07,
      "loss": 0.1099,
      "num_tokens": 270470318.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.16434319969266653,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 1570
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1581.8021240234375,
      "epoch": 0.6033605376860298,
      "grad_norm": 0.248046875,
      "learning_rate": 4.66579858640558e-07,
      "loss": 0.0902,
      "num_tokens": 270637563.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.3184647783637047,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1571
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.7500610351562,
      "epoch": 0.6037445991358618,
      "grad_norm": 0.251953125,
      "learning_rate": 4.6598676003519634e-07,
      "loss": 0.088,
      "num_tokens": 270805131.0,
      "reward": 0.5416666865348816,
      "reward_std": 0.26297792978584766,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.5937805175781,
      "epoch": 0.6041286605856937,
      "grad_norm": 0.2177734375,
      "learning_rate": 4.653938126026261e-07,
      "loss": 0.0733,
      "num_tokens": 270984596.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.22644210141152143,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1497.9687805175781,
      "epoch": 0.6045127220355256,
      "grad_norm": 0.2119140625,
      "learning_rate": 4.648010174097923e-07,
      "loss": 0.1218,
      "num_tokens": 271144297.0,
      "reward": 0.5729166865348816,
      "reward_std": 0.25668954476714134,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.489583320915699,
      "step": 1574
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.2292175292969,
      "epoch": 0.6048967834853577,
      "grad_norm": 0.201171875,
      "learning_rate": 4.6420837552336634e-07,
      "loss": 0.089,
      "num_tokens": 271327335.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.16858358681201935,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1575
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1287.7396240234375,
      "epoch": 0.6052808449351896,
      "grad_norm": 0.275390625,
      "learning_rate": 4.636158880097434e-07,
      "loss": 0.1198,
      "num_tokens": 271464222.0,
      "reward": 0.6770833432674408,
      "reward_std": 0.39077191799879074,
      "rewards/accuracy_reward": 0.17708333022892475,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4999999925494194,
      "step": 1576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1609.5000305175781,
      "epoch": 0.6056649063850216,
      "grad_norm": 0.201171875,
      "learning_rate": 4.6302355593504116e-07,
      "loss": 0.1137,
      "num_tokens": 271633974.0,
      "reward": 0.6562500149011612,
      "reward_std": 0.31752143427729607,
      "rewards/accuracy_reward": 0.2500000027939677,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.0937805175781,
      "epoch": 0.6060489678348536,
      "grad_norm": 0.2109375,
      "learning_rate": 4.624313803650973e-07,
      "loss": 0.1466,
      "num_tokens": 271818223.0,
      "reward": 0.3750000074505806,
      "reward_std": 0.17968551069498062,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666716337204,
      "step": 1578
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.3750305175781,
      "epoch": 0.6064330292846856,
      "grad_norm": 0.27734375,
      "learning_rate": 4.618393623654685e-07,
      "loss": 0.153,
      "num_tokens": 271993435.0,
      "reward": 0.3932291716337204,
      "reward_std": 0.1598371621221304,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 1579
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.3854675292969,
      "epoch": 0.6068170907345175,
      "grad_norm": 0.21484375,
      "learning_rate": 4.612475030014271e-07,
      "loss": 0.1236,
      "num_tokens": 272163688.0,
      "reward": 0.5520833432674408,
      "reward_std": 0.29246312379837036,
      "rewards/accuracy_reward": 0.11458333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 1580
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.4167175292969,
      "epoch": 0.6072011521843494,
      "grad_norm": 0.1845703125,
      "learning_rate": 4.606558033379605e-07,
      "loss": 0.1075,
      "num_tokens": 272335128.0,
      "reward": 0.520833358168602,
      "reward_std": 0.209262877702713,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 1581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.9271240234375,
      "epoch": 0.6075852136341815,
      "grad_norm": 0.2490234375,
      "learning_rate": 4.600642644397688e-07,
      "loss": 0.0648,
      "num_tokens": 272502657.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.13412399776279926,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1582
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.6354675292969,
      "epoch": 0.6079692750840134,
      "grad_norm": 0.259765625,
      "learning_rate": 4.5947288737126223e-07,
      "loss": 0.178,
      "num_tokens": 272672302.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.19778116419911385,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1583
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.3750305175781,
      "epoch": 0.6083533365338454,
      "grad_norm": 0.216796875,
      "learning_rate": 4.588816731965607e-07,
      "loss": 0.0459,
      "num_tokens": 272851658.0,
      "reward": 0.4635416865348816,
      "reward_std": 0.15595772117376328,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1887.7084045410156,
      "epoch": 0.6087373979836774,
      "grad_norm": 0.1689453125,
      "learning_rate": 4.5829062297949004e-07,
      "loss": 0.069,
      "num_tokens": 273047870.0,
      "reward": 0.3828125074505806,
      "reward_std": 0.15395398065447807,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 1585
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.104248046875,
      "epoch": 0.6091214594335094,
      "grad_norm": 0.2275390625,
      "learning_rate": 4.5769973778358216e-07,
      "loss": 0.1313,
      "num_tokens": 273233888.0,
      "reward": 0.3723958432674408,
      "reward_std": 0.1560707502067089,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625074505806,
      "step": 1586
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.7708740234375,
      "epoch": 0.6095055208833413,
      "grad_norm": 0.1875,
      "learning_rate": 4.5710901867207086e-07,
      "loss": 0.0708,
      "num_tokens": 273420506.0,
      "reward": 0.4192708507180214,
      "reward_std": 0.2039514072239399,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1587
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.3021240234375,
      "epoch": 0.6098895823331733,
      "grad_norm": 0.30078125,
      "learning_rate": 4.5651846670789207e-07,
      "loss": 0.1391,
      "num_tokens": 273587287.0,
      "reward": 0.549479179084301,
      "reward_std": 0.22420576959848404,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140624925494194,
      "step": 1588
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.8542175292969,
      "epoch": 0.6102736437830053,
      "grad_norm": 0.21875,
      "learning_rate": 4.5592808295368034e-07,
      "loss": 0.1294,
      "num_tokens": 273747217.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.20435348339378834,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1478.6771240234375,
      "epoch": 0.6106577052328372,
      "grad_norm": 0.1962890625,
      "learning_rate": 4.553378684717679e-07,
      "loss": 0.0536,
      "num_tokens": 273904418.0,
      "reward": 0.4531250074505806,
      "reward_std": 0.15068975370377302,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 1590
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1299.5104675292969,
      "epoch": 0.6110417666826692,
      "grad_norm": 0.259765625,
      "learning_rate": 4.54747824324182e-07,
      "loss": 0.1146,
      "num_tokens": 274042955.0,
      "reward": 0.6302083507180214,
      "reward_std": 0.2551770322024822,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5052083283662796,
      "step": 1591
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.4271240234375,
      "epoch": 0.6114258281325012,
      "grad_norm": 0.263671875,
      "learning_rate": 4.541579515726439e-07,
      "loss": 0.119,
      "num_tokens": 274208844.0,
      "reward": 0.5026041939854622,
      "reward_std": 0.2526118718087673,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.7083740234375,
      "epoch": 0.6118098895823332,
      "grad_norm": 0.1875,
      "learning_rate": 4.535682512785659e-07,
      "loss": 0.0476,
      "num_tokens": 274390376.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.12067150417715311,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 1593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.322998046875,
      "epoch": 0.6121939510321651,
      "grad_norm": 0.2001953125,
      "learning_rate": 4.5297872450305064e-07,
      "loss": 0.0839,
      "num_tokens": 274573703.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.20152715221047401,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1803.3125610351562,
      "epoch": 0.6125780124819972,
      "grad_norm": 0.19140625,
      "learning_rate": 4.523893723068878e-07,
      "loss": 0.0707,
      "num_tokens": 274762365.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.17569342255592346,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 1595
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1483.2604675292969,
      "epoch": 0.6129620739318291,
      "grad_norm": 0.2275390625,
      "learning_rate": 4.518001957505535e-07,
      "loss": 0.0773,
      "num_tokens": 274919062.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.23437261208891869,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 1596
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1642.8229675292969,
      "epoch": 0.613346135381661,
      "grad_norm": 0.26953125,
      "learning_rate": 4.512111958942076e-07,
      "loss": 0.0506,
      "num_tokens": 275090645.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.17354169115424156,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 1597
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.2917175292969,
      "epoch": 0.613730196831493,
      "grad_norm": 0.236328125,
      "learning_rate": 4.50622373797692e-07,
      "loss": 0.1136,
      "num_tokens": 275253097.0,
      "reward": 0.440104179084301,
      "reward_std": 0.17356998659670353,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1598
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1440.3334045410156,
      "epoch": 0.614114258281325,
      "grad_norm": 0.2216796875,
      "learning_rate": 4.5003373052052863e-07,
      "loss": 0.078,
      "num_tokens": 275404617.0,
      "reward": 0.5677083432674408,
      "reward_std": 0.2381473844870925,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416716337204,
      "step": 1599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.4375610351562,
      "epoch": 0.614498319731157,
      "grad_norm": 0.181640625,
      "learning_rate": 4.4944526712191777e-07,
      "loss": 0.0907,
      "num_tokens": 275586219.0,
      "reward": 0.408854179084301,
      "reward_std": 0.1908118948340416,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 1600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1482.96875,
      "epoch": 0.6148823811809889,
      "grad_norm": 0.2314453125,
      "learning_rate": 4.488569846607362e-07,
      "loss": 0.0877,
      "num_tokens": 275741208.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.30846062675118446,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687500074505806,
      "step": 1601
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.0833740234375,
      "epoch": 0.615266442630821,
      "grad_norm": 0.2490234375,
      "learning_rate": 4.4826888419553475e-07,
      "loss": 0.0611,
      "num_tokens": 275897744.0,
      "reward": 0.494791679084301,
      "reward_std": 0.20661808922886848,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 1602
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1386.6979370117188,
      "epoch": 0.6156505040806529,
      "grad_norm": 0.251953125,
      "learning_rate": 4.476809667845373e-07,
      "loss": 0.1114,
      "num_tokens": 276048507.0,
      "reward": 0.6328125223517418,
      "reward_std": 0.2292515691369772,
      "rewards/accuracy_reward": 0.17708333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291641831398,
      "step": 1603
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.0208740234375,
      "epoch": 0.6160345655304849,
      "grad_norm": 0.21875,
      "learning_rate": 4.4709323348563754e-07,
      "loss": 0.1254,
      "num_tokens": 276216325.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.19388050213456154,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 1604
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.9271240234375,
      "epoch": 0.6164186269803168,
      "grad_norm": 0.1904296875,
      "learning_rate": 4.4650568535639865e-07,
      "loss": 0.0685,
      "num_tokens": 276398878.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.26508947741240263,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1605
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.9896240234375,
      "epoch": 0.6168026884301488,
      "grad_norm": 0.18359375,
      "learning_rate": 4.4591832345405023e-07,
      "loss": 0.0994,
      "num_tokens": 276561637.0,
      "reward": 0.401041679084301,
      "reward_std": 0.08757995907217264,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 1606
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1680.5833740234375,
      "epoch": 0.6171867498799808,
      "grad_norm": 0.2080078125,
      "learning_rate": 4.4533114883548695e-07,
      "loss": 0.1175,
      "num_tokens": 276737829.0,
      "reward": 0.5312500074505806,
      "reward_std": 0.2575953211635351,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1607
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.791748046875,
      "epoch": 0.6175708113298127,
      "grad_norm": 0.19921875,
      "learning_rate": 4.447441625572662e-07,
      "loss": 0.0915,
      "num_tokens": 276910057.0,
      "reward": 0.5130208507180214,
      "reward_std": 0.20800169929862022,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 1608
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1702.0312805175781,
      "epoch": 0.6179548727796448,
      "grad_norm": 0.265625,
      "learning_rate": 4.441573656756069e-07,
      "loss": 0.1133,
      "num_tokens": 277088444.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.233688086271286,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1609
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.4167175292969,
      "epoch": 0.6183389342294767,
      "grad_norm": 0.212890625,
      "learning_rate": 4.4357075924638674e-07,
      "loss": 0.0959,
      "num_tokens": 277259332.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.20801956206560135,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1610
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.3750610351562,
      "epoch": 0.6187229956793087,
      "grad_norm": 0.205078125,
      "learning_rate": 4.42984344325141e-07,
      "loss": 0.0999,
      "num_tokens": 277435000.0,
      "reward": 0.5234375223517418,
      "reward_std": 0.278215866535902,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 1611
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1789.2604675292969,
      "epoch": 0.6191070571291407,
      "grad_norm": 0.23046875,
      "learning_rate": 4.423981219670604e-07,
      "loss": 0.116,
      "num_tokens": 277621401.0,
      "reward": 0.3750000149011612,
      "reward_std": 0.15840032324194908,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 1612
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1579.5521240234375,
      "epoch": 0.6194911185789727,
      "grad_norm": 0.23828125,
      "learning_rate": 4.4181209322698877e-07,
      "loss": 0.0879,
      "num_tokens": 277787878.0,
      "reward": 0.502604179084301,
      "reward_std": 0.23085936158895493,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 1613
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.4687805175781,
      "epoch": 0.6198751800288046,
      "grad_norm": 0.208984375,
      "learning_rate": 4.412262591594221e-07,
      "loss": 0.1122,
      "num_tokens": 277974691.0,
      "reward": 0.3880208507180214,
      "reward_std": 0.18167823553085327,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1614
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1450.2604675292969,
      "epoch": 0.6202592414786365,
      "grad_norm": 0.244140625,
      "learning_rate": 4.406406208185056e-07,
      "loss": 0.07,
      "num_tokens": 278129004.0,
      "reward": 0.658854179084301,
      "reward_std": 0.3064562976360321,
      "rewards/accuracy_reward": 0.2135416679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 1615
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.5417175292969,
      "epoch": 0.6206433029284686,
      "grad_norm": 0.2236328125,
      "learning_rate": 4.400551792580326e-07,
      "loss": 0.1919,
      "num_tokens": 278297952.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.1739494502544403,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 1616
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1496.7604370117188,
      "epoch": 0.6210273643783005,
      "grad_norm": 0.232421875,
      "learning_rate": 4.3946993553144206e-07,
      "loss": 0.1097,
      "num_tokens": 278456665.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.2027148026973009,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1617
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.0625610351562,
      "epoch": 0.6214114258281325,
      "grad_norm": 0.2109375,
      "learning_rate": 4.3888489069181747e-07,
      "loss": 0.098,
      "num_tokens": 278630223.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.15975597128272057,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 1618
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.7812805175781,
      "epoch": 0.6217954872779645,
      "grad_norm": 0.216796875,
      "learning_rate": 4.3830004579188364e-07,
      "loss": 0.0733,
      "num_tokens": 278788194.0,
      "reward": 0.4921875074505806,
      "reward_std": 0.21058867871761322,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1619
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.1666870117188,
      "epoch": 0.6221795487277965,
      "grad_norm": 0.1982421875,
      "learning_rate": 4.377154018840066e-07,
      "loss": 0.1375,
      "num_tokens": 278970218.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.182465685531497,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083432674408,
      "step": 1620
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.9062805175781,
      "epoch": 0.6225636101776284,
      "grad_norm": 0.2265625,
      "learning_rate": 4.3713096002018965e-07,
      "loss": 0.1273,
      "num_tokens": 279137081.0,
      "reward": 0.518229179084301,
      "reward_std": 0.21236559748649597,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1621
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1609.3958435058594,
      "epoch": 0.6229476716274603,
      "grad_norm": 0.205078125,
      "learning_rate": 4.365467212520736e-07,
      "loss": 0.0846,
      "num_tokens": 279306215.0,
      "reward": 0.4270833358168602,
      "reward_std": 0.141995245590806,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1622
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1779.5938110351562,
      "epoch": 0.6233317330772924,
      "grad_norm": 0.203125,
      "learning_rate": 4.359626866309329e-07,
      "loss": 0.0833,
      "num_tokens": 279492448.0,
      "reward": 0.4739583507180214,
      "reward_std": 0.21347392350435257,
      "rewards/accuracy_reward": 0.10416666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1623
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.5729370117188,
      "epoch": 0.6237157945271243,
      "grad_norm": 0.1806640625,
      "learning_rate": 4.3537885720767533e-07,
      "loss": 0.054,
      "num_tokens": 279661063.0,
      "reward": 0.424479179084301,
      "reward_std": 0.14471285324543715,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1624
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.3958740234375,
      "epoch": 0.6240998559769563,
      "grad_norm": 0.2001953125,
      "learning_rate": 4.34795234032839e-07,
      "loss": 0.1222,
      "num_tokens": 279828709.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.22351068258285522,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1625
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.9791870117188,
      "epoch": 0.6244839174267883,
      "grad_norm": 0.2275390625,
      "learning_rate": 4.3421181815659093e-07,
      "loss": 0.102,
      "num_tokens": 279996579.0,
      "reward": 0.5703125223517418,
      "reward_std": 0.2271323762834072,
      "rewards/accuracy_reward": 0.15624999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140624925494194,
      "step": 1626
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.4791870117188,
      "epoch": 0.6248679788766203,
      "grad_norm": 0.197265625,
      "learning_rate": 4.336286106287256e-07,
      "loss": 0.0778,
      "num_tokens": 280169049.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.14579606894403696,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1627
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.1875305175781,
      "epoch": 0.6252520403264522,
      "grad_norm": 0.21484375,
      "learning_rate": 4.33045612498662e-07,
      "loss": 0.1001,
      "num_tokens": 280333459.0,
      "reward": 0.567708358168602,
      "reward_std": 0.27394726127386093,
      "rewards/accuracy_reward": 0.14583333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1628
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.3334045410156,
      "epoch": 0.6256361017762843,
      "grad_norm": 0.265625,
      "learning_rate": 4.324628248154426e-07,
      "loss": 0.1668,
      "num_tokens": 280495763.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.28185899555683136,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 1629
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.6354370117188,
      "epoch": 0.6260201632261162,
      "grad_norm": 0.1552734375,
      "learning_rate": 4.318802486277312e-07,
      "loss": 0.0716,
      "num_tokens": 280665024.0,
      "reward": 0.4036458507180214,
      "reward_std": 0.103530989959836,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1630
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.7604675292969,
      "epoch": 0.6264042246759481,
      "grad_norm": 0.234375,
      "learning_rate": 4.3129788498381114e-07,
      "loss": 0.0748,
      "num_tokens": 280822129.0,
      "reward": 0.5807291865348816,
      "reward_std": 0.31163972429931164,
      "rewards/accuracy_reward": 0.16666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1631
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.1250305175781,
      "epoch": 0.6267882861257801,
      "grad_norm": 0.2158203125,
      "learning_rate": 4.3071573493158284e-07,
      "loss": 0.0711,
      "num_tokens": 280992949.0,
      "reward": 0.518229179084301,
      "reward_std": 0.22736458852887154,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1632
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1453.5104370117188,
      "epoch": 0.6271723475756121,
      "grad_norm": 0.2265625,
      "learning_rate": 4.301337995185631e-07,
      "loss": 0.0882,
      "num_tokens": 281143542.0,
      "reward": 0.536458358168602,
      "reward_std": 0.18086699396371841,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.463541679084301,
      "step": 1633
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.8646240234375,
      "epoch": 0.6275564090254441,
      "grad_norm": 0.224609375,
      "learning_rate": 4.295520797918818e-07,
      "loss": 0.0922,
      "num_tokens": 281306745.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.2314123921096325,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1634
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.1250610351562,
      "epoch": 0.627940470475276,
      "grad_norm": 0.1748046875,
      "learning_rate": 4.289705767982813e-07,
      "loss": 0.0599,
      "num_tokens": 281494501.0,
      "reward": 0.440104179084301,
      "reward_std": 0.1973125645890832,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875074505806,
      "step": 1635
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.6250305175781,
      "epoch": 0.6283245319251081,
      "grad_norm": 0.2197265625,
      "learning_rate": 4.283892915841134e-07,
      "loss": 0.1135,
      "num_tokens": 281677961.0,
      "reward": 0.3984375074505806,
      "reward_std": 0.14303739555180073,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1636
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.7083740234375,
      "epoch": 0.62870859337494,
      "grad_norm": 0.2353515625,
      "learning_rate": 4.2780822519533875e-07,
      "loss": 0.1367,
      "num_tokens": 281846413.0,
      "reward": 0.432291679084301,
      "reward_std": 0.19268380478024483,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1637
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.3646240234375,
      "epoch": 0.629092654824772,
      "grad_norm": 0.232421875,
      "learning_rate": 4.2722737867752346e-07,
      "loss": 0.1391,
      "num_tokens": 282021960.0,
      "reward": 0.502604179084301,
      "reward_std": 0.24953152611851692,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.440104179084301,
      "step": 1638
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1707.6146240234375,
      "epoch": 0.6294767162746039,
      "grad_norm": 0.228515625,
      "learning_rate": 4.266467530758386e-07,
      "loss": 0.1099,
      "num_tokens": 282201939.0,
      "reward": 0.440104179084301,
      "reward_std": 0.2200176827609539,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 1639
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1413.4687805175781,
      "epoch": 0.6298607777244359,
      "grad_norm": 0.25,
      "learning_rate": 4.2606634943505734e-07,
      "loss": 0.1046,
      "num_tokens": 282354168.0,
      "reward": 0.580729179084301,
      "reward_std": 0.3806755840778351,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125074505806,
      "step": 1640
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1770.3646240234375,
      "epoch": 0.6302448391742679,
      "grad_norm": 0.15234375,
      "learning_rate": 4.254861687995538e-07,
      "loss": 0.0561,
      "num_tokens": 282539963.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.17728472594171762,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1641
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.9896240234375,
      "epoch": 0.6306289006240998,
      "grad_norm": 0.2158203125,
      "learning_rate": 4.249062122133007e-07,
      "loss": 0.0823,
      "num_tokens": 282710322.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.1307335253804922,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1642
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.5208740234375,
      "epoch": 0.6310129620739319,
      "grad_norm": 0.2451171875,
      "learning_rate": 4.243264807198671e-07,
      "loss": 0.1313,
      "num_tokens": 282875252.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.20415166392922401,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1643
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1387.8958435058594,
      "epoch": 0.6313970235237638,
      "grad_norm": 0.291015625,
      "learning_rate": 4.237469753624181e-07,
      "loss": 0.0857,
      "num_tokens": 283021626.0,
      "reward": 0.6015625223517418,
      "reward_std": 0.28142622113227844,
      "rewards/accuracy_reward": 0.13541666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 1644
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.9479675292969,
      "epoch": 0.6317810849735958,
      "grad_norm": 0.2421875,
      "learning_rate": 4.231676971837108e-07,
      "loss": 0.1405,
      "num_tokens": 283198653.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.15359573252499104,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1645
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.6667175292969,
      "epoch": 0.6321651464234278,
      "grad_norm": 0.216796875,
      "learning_rate": 4.225886472260942e-07,
      "loss": 0.1085,
      "num_tokens": 283366629.0,
      "reward": 0.653645858168602,
      "reward_std": 0.37632719427347183,
      "rewards/accuracy_reward": 0.21874999441206455,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1646
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.5000305175781,
      "epoch": 0.6325492078732597,
      "grad_norm": 0.2119140625,
      "learning_rate": 4.220098265315065e-07,
      "loss": 0.076,
      "num_tokens": 283531829.0,
      "reward": 0.4687500074505806,
      "reward_std": 0.17871609888970852,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 1647
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.5312805175781,
      "epoch": 0.6329332693230917,
      "grad_norm": 0.2431640625,
      "learning_rate": 4.2143123614147316e-07,
      "loss": 0.0841,
      "num_tokens": 283699408.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.29437412321567535,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583432674408,
      "step": 1648
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.7708740234375,
      "epoch": 0.6333173307729236,
      "grad_norm": 0.2314453125,
      "learning_rate": 4.2085287709710525e-07,
      "loss": 0.0568,
      "num_tokens": 283867250.0,
      "reward": 0.463541679084301,
      "reward_std": 0.18740024231374264,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1649
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.7292175292969,
      "epoch": 0.6337013922227557,
      "grad_norm": 0.2314453125,
      "learning_rate": 4.202747504390981e-07,
      "loss": 0.0593,
      "num_tokens": 284030984.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.2388296891003847,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 1650
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.3646240234375,
      "epoch": 0.6340854536725876,
      "grad_norm": 0.228515625,
      "learning_rate": 4.1969685720772807e-07,
      "loss": 0.0998,
      "num_tokens": 284197963.0,
      "reward": 0.432291679084301,
      "reward_std": 0.18635015562176704,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1651
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.0208740234375,
      "epoch": 0.6344695151224196,
      "grad_norm": 0.2275390625,
      "learning_rate": 4.19119198442852e-07,
      "loss": 0.1327,
      "num_tokens": 284364221.0,
      "reward": 0.5911458432674408,
      "reward_std": 0.30909042060375214,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 1652
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.7604370117188,
      "epoch": 0.6348535765722516,
      "grad_norm": 0.205078125,
      "learning_rate": 4.1854177518390474e-07,
      "loss": 0.0866,
      "num_tokens": 284534630.0,
      "reward": 0.505208358168602,
      "reward_std": 0.17198907490819693,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 1653
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1537.6250305175781,
      "epoch": 0.6352376380220836,
      "grad_norm": 0.240234375,
      "learning_rate": 4.1796458846989755e-07,
      "loss": 0.1126,
      "num_tokens": 284695338.0,
      "reward": 0.5937500074505806,
      "reward_std": 0.2242546258494258,
      "rewards/accuracy_reward": 0.1874999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 1654
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.9063110351562,
      "epoch": 0.6356216994719155,
      "grad_norm": 0.25,
      "learning_rate": 4.1738763933941554e-07,
      "loss": 0.1122,
      "num_tokens": 284848433.0,
      "reward": 0.4843750223517418,
      "reward_std": 0.19563505426049232,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 1655
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.5625305175781,
      "epoch": 0.6360057609217474,
      "grad_norm": 0.2255859375,
      "learning_rate": 4.168109288306165e-07,
      "loss": 0.1054,
      "num_tokens": 285017127.0,
      "reward": 0.416666679084301,
      "reward_std": 0.1539493389427662,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1656
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1772.3229675292969,
      "epoch": 0.6363898223715795,
      "grad_norm": 0.208984375,
      "learning_rate": 4.162344579812298e-07,
      "loss": 0.0772,
      "num_tokens": 285202142.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.23477165959775448,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 1657
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.3542175292969,
      "epoch": 0.6367738838214114,
      "grad_norm": 0.224609375,
      "learning_rate": 4.156582278285518e-07,
      "loss": 0.1005,
      "num_tokens": 285370952.0,
      "reward": 0.5468750074505806,
      "reward_std": 0.21544573828577995,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1658
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1819.9166870117188,
      "epoch": 0.6371579452712434,
      "grad_norm": 0.1728515625,
      "learning_rate": 4.150822394094473e-07,
      "loss": 0.0734,
      "num_tokens": 285560312.0,
      "reward": 0.3984375149011612,
      "reward_std": 0.14799688942730427,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 1659
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.3125610351562,
      "epoch": 0.6375420067210754,
      "grad_norm": 0.2392578125,
      "learning_rate": 4.1450649376034496e-07,
      "loss": 0.1418,
      "num_tokens": 285722326.0,
      "reward": 0.5182291939854622,
      "reward_std": 0.24438812490552664,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1660
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.4584045410156,
      "epoch": 0.6379260681709074,
      "grad_norm": 0.1826171875,
      "learning_rate": 4.139309919172379e-07,
      "loss": 0.1016,
      "num_tokens": 285906802.0,
      "reward": 0.3828125074505806,
      "reward_std": 0.15351818781346083,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 1661
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.3438110351562,
      "epoch": 0.6383101296207393,
      "grad_norm": 0.2021484375,
      "learning_rate": 4.133557349156791e-07,
      "loss": 0.1074,
      "num_tokens": 286079787.0,
      "reward": 0.4088541716337204,
      "reward_std": 0.12718106620013714,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1662
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1433.0625305175781,
      "epoch": 0.6386941910705712,
      "grad_norm": 0.265625,
      "learning_rate": 4.1278072379078233e-07,
      "loss": 0.1016,
      "num_tokens": 286231673.0,
      "reward": 0.6015625074505806,
      "reward_std": 0.3164159841835499,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625074505806,
      "step": 1663
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.4167175292969,
      "epoch": 0.6390782525204033,
      "grad_norm": 0.2099609375,
      "learning_rate": 4.122059595772177e-07,
      "loss": 0.0724,
      "num_tokens": 286396457.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.18904619570821524,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1664
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1428.0208435058594,
      "epoch": 0.6394623139702352,
      "grad_norm": 0.2099609375,
      "learning_rate": 4.1163144330921206e-07,
      "loss": 0.1245,
      "num_tokens": 286546699.0,
      "reward": 0.7500000149011612,
      "reward_std": 0.317841999232769,
      "rewards/accuracy_reward": 0.29166666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333432674408,
      "step": 1665
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1704.9063110351562,
      "epoch": 0.6398463754200672,
      "grad_norm": 0.201171875,
      "learning_rate": 4.1105717602054526e-07,
      "loss": 0.0657,
      "num_tokens": 286723938.0,
      "reward": 0.510416679084301,
      "reward_std": 0.27393022924661636,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1666
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.1041870117188,
      "epoch": 0.6402304368698992,
      "grad_norm": 0.2158203125,
      "learning_rate": 4.1048315874454977e-07,
      "loss": 0.1105,
      "num_tokens": 286897484.0,
      "reward": 0.447916679084301,
      "reward_std": 0.1987963281571865,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1667
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1839.0417175292969,
      "epoch": 0.6406144983197312,
      "grad_norm": 0.1884765625,
      "learning_rate": 4.0990939251410793e-07,
      "loss": 0.0832,
      "num_tokens": 287089136.0,
      "reward": 0.3489583432674408,
      "reward_std": 0.1129368944093585,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416641831398,
      "step": 1668
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1551.7917175292969,
      "epoch": 0.6409985597695631,
      "grad_norm": 0.2158203125,
      "learning_rate": 4.093358783616504e-07,
      "loss": 0.1551,
      "num_tokens": 287252596.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.16677107568830252,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1669
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.6354675292969,
      "epoch": 0.6413826212193952,
      "grad_norm": 0.2275390625,
      "learning_rate": 4.0876261731915405e-07,
      "loss": 0.0914,
      "num_tokens": 287418449.0,
      "reward": 0.5390625074505806,
      "reward_std": 0.23166047036647797,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1670
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.1771240234375,
      "epoch": 0.6417666826692271,
      "grad_norm": 0.24609375,
      "learning_rate": 4.081896104181405e-07,
      "loss": 0.1068,
      "num_tokens": 287585682.0,
      "reward": 0.5078125223517418,
      "reward_std": 0.22243047319352627,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1671
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.4896240234375,
      "epoch": 0.642150744119059,
      "grad_norm": 0.2216796875,
      "learning_rate": 4.076168586896744e-07,
      "loss": 0.0789,
      "num_tokens": 287760289.0,
      "reward": 0.4817708432674408,
      "reward_std": 0.2660445738583803,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 1672
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.5000305175781,
      "epoch": 0.642534805568891,
      "grad_norm": 0.2373046875,
      "learning_rate": 4.070443631643605e-07,
      "loss": 0.039,
      "num_tokens": 287926753.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.16485037561506033,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1673
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1617.6250305175781,
      "epoch": 0.642918867018723,
      "grad_norm": 0.2041015625,
      "learning_rate": 4.064721248723434e-07,
      "loss": 0.0793,
      "num_tokens": 288097045.0,
      "reward": 0.440104179084301,
      "reward_std": 0.14256038423627615,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 1674
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.4479370117188,
      "epoch": 0.643302928468555,
      "grad_norm": 0.2138671875,
      "learning_rate": 4.059001448433038e-07,
      "loss": 0.1045,
      "num_tokens": 288272888.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.20835494808852673,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1675
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1540.4271240234375,
      "epoch": 0.6436869899183869,
      "grad_norm": 0.2373046875,
      "learning_rate": 4.0532842410645874e-07,
      "loss": 0.1043,
      "num_tokens": 288437929.0,
      "reward": 0.486979179084301,
      "reward_std": 0.2278006449341774,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 1676
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.2292175292969,
      "epoch": 0.644071051368219,
      "grad_norm": 0.2021484375,
      "learning_rate": 4.0475696369055767e-07,
      "loss": 0.1435,
      "num_tokens": 288609663.0,
      "reward": 0.4401041716337204,
      "reward_std": 0.20194425247609615,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 1677
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1432.1875,
      "epoch": 0.6444551128180509,
      "grad_norm": 0.23828125,
      "learning_rate": 4.0418576462388287e-07,
      "loss": 0.1071,
      "num_tokens": 288760865.0,
      "reward": 0.692708358168602,
      "reward_std": 0.3283724933862686,
      "rewards/accuracy_reward": 0.25,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 1678
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.8020935058594,
      "epoch": 0.6448391742678828,
      "grad_norm": 0.1923828125,
      "learning_rate": 4.036148279342448e-07,
      "loss": 0.1267,
      "num_tokens": 288944710.0,
      "reward": 0.393229179084301,
      "reward_std": 0.17754245176911354,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1679
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.2083740234375,
      "epoch": 0.6452232357177148,
      "grad_norm": 0.1865234375,
      "learning_rate": 4.030441546489837e-07,
      "loss": 0.0669,
      "num_tokens": 289109578.0,
      "reward": 0.5130208656191826,
      "reward_std": 0.22389247454702854,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1680
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.6145935058594,
      "epoch": 0.6456072971675468,
      "grad_norm": 0.2197265625,
      "learning_rate": 4.024737457949636e-07,
      "loss": 0.0638,
      "num_tokens": 289283901.0,
      "reward": 0.4583333507180214,
      "reward_std": 0.1265910156071186,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 1681
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1379.7292175292969,
      "epoch": 0.6459913586173788,
      "grad_norm": 0.2333984375,
      "learning_rate": 4.019036023985749e-07,
      "loss": 0.0938,
      "num_tokens": 289431059.0,
      "reward": 0.6875000298023224,
      "reward_std": 0.27673428505659103,
      "rewards/accuracy_reward": 0.2395833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 1682
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1387.3854675292969,
      "epoch": 0.6463754200672107,
      "grad_norm": 0.248046875,
      "learning_rate": 4.0133372548572863e-07,
      "loss": 0.0777,
      "num_tokens": 289579816.0,
      "reward": 0.6015625223517418,
      "reward_std": 0.22697646170854568,
      "rewards/accuracy_reward": 0.15624999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453124925494194,
      "step": 1683
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1361.6458892822266,
      "epoch": 0.6467594815170428,
      "grad_norm": 0.296875,
      "learning_rate": 4.007641160818575e-07,
      "loss": 0.0672,
      "num_tokens": 289725814.0,
      "reward": 0.5338541865348816,
      "reward_std": 0.22355790995061398,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541641831398,
      "step": 1684
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1477.4792175292969,
      "epoch": 0.6471435429668747,
      "grad_norm": 0.265625,
      "learning_rate": 4.0019477521191213e-07,
      "loss": 0.1025,
      "num_tokens": 289884572.0,
      "reward": 0.479166679084301,
      "reward_std": 0.16824331134557724,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 1685
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1731.0416870117188,
      "epoch": 0.6475276044167066,
      "grad_norm": 0.201171875,
      "learning_rate": 3.996257039003602e-07,
      "loss": 0.0775,
      "num_tokens": 290064568.0,
      "reward": 0.424479179084301,
      "reward_std": 0.15589500963687897,
      "rewards/accuracy_reward": 0.08854166697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 1686
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.9375610351562,
      "epoch": 0.6479116658665387,
      "grad_norm": 0.23828125,
      "learning_rate": 3.990569031711845e-07,
      "loss": 0.078,
      "num_tokens": 290242210.0,
      "reward": 0.4973958358168602,
      "reward_std": 0.1822125930339098,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 1687
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1506.9375305175781,
      "epoch": 0.6482957273163706,
      "grad_norm": 0.193359375,
      "learning_rate": 3.984883740478804e-07,
      "loss": 0.0439,
      "num_tokens": 290399172.0,
      "reward": 0.5390625149011612,
      "reward_std": 0.2368123922497034,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 1688
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.3125,
      "epoch": 0.6486797887662026,
      "grad_norm": 0.2412109375,
      "learning_rate": 3.9792011755345557e-07,
      "loss": 0.1231,
      "num_tokens": 290557954.0,
      "reward": 0.6015625149011612,
      "reward_std": 0.3027395196259022,
      "rewards/accuracy_reward": 0.18750000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1689
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1269.5833740234375,
      "epoch": 0.6490638502160345,
      "grad_norm": 0.302734375,
      "learning_rate": 3.973521347104258e-07,
      "loss": 0.1113,
      "num_tokens": 290695138.0,
      "reward": 0.6276041865348816,
      "reward_std": 0.2979467734694481,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875,
      "step": 1690
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.6979370117188,
      "epoch": 0.6494479116658666,
      "grad_norm": 0.2138671875,
      "learning_rate": 3.9678442654081555e-07,
      "loss": 0.1025,
      "num_tokens": 290876301.0,
      "reward": 0.408854179084301,
      "reward_std": 0.17111915536224842,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1691
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1572.2708740234375,
      "epoch": 0.6498319731156985,
      "grad_norm": 0.2431640625,
      "learning_rate": 3.9621699406615437e-07,
      "loss": 0.1093,
      "num_tokens": 291041615.0,
      "reward": 0.572916679084301,
      "reward_std": 0.2589208874851465,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687500074505806,
      "step": 1692
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.0416870117188,
      "epoch": 0.6502160345655305,
      "grad_norm": 0.21484375,
      "learning_rate": 3.956498383074764e-07,
      "loss": 0.092,
      "num_tokens": 291205971.0,
      "reward": 0.4453125223517418,
      "reward_std": 0.1863338234834373,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1693
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.3750305175781,
      "epoch": 0.6506000960153625,
      "grad_norm": 0.166015625,
      "learning_rate": 3.9508296028531686e-07,
      "loss": 0.0692,
      "num_tokens": 291389567.0,
      "reward": 0.3723958432674408,
      "reward_std": 0.13719091145321727,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 1694
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1795.1146240234375,
      "epoch": 0.6509841574651944,
      "grad_norm": 0.220703125,
      "learning_rate": 3.945163610197124e-07,
      "loss": 0.1154,
      "num_tokens": 291577962.0,
      "reward": 0.4479166716337204,
      "reward_std": 0.27814973145723343,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1695
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.0937805175781,
      "epoch": 0.6513682189150264,
      "grad_norm": 0.2490234375,
      "learning_rate": 3.939500415301968e-07,
      "loss": 0.1171,
      "num_tokens": 291743243.0,
      "reward": 0.5494791939854622,
      "reward_std": 0.22787725180387497,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1696
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.6770935058594,
      "epoch": 0.6517522803648583,
      "grad_norm": 0.232421875,
      "learning_rate": 3.933840028358019e-07,
      "loss": 0.1027,
      "num_tokens": 291913444.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.19632617384195328,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1697
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1707.4792175292969,
      "epoch": 0.6521363418146904,
      "grad_norm": 0.1787109375,
      "learning_rate": 3.9281824595505255e-07,
      "loss": 0.0846,
      "num_tokens": 292091794.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.21553528122603893,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 1698
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.4375610351562,
      "epoch": 0.6525204032645223,
      "grad_norm": 0.26953125,
      "learning_rate": 3.9225277190596805e-07,
      "loss": 0.1021,
      "num_tokens": 292257500.0,
      "reward": 0.5286458432674408,
      "reward_std": 0.25324851274490356,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1699
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.3958740234375,
      "epoch": 0.6529044647143543,
      "grad_norm": 0.193359375,
      "learning_rate": 3.9168758170605807e-07,
      "loss": 0.0789,
      "num_tokens": 292429058.0,
      "reward": 0.489583358168602,
      "reward_std": 0.19108237884938717,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4374999925494194,
      "step": 1700
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.0417175292969,
      "epoch": 0.6532885261641863,
      "grad_norm": 0.1982421875,
      "learning_rate": 3.911226763723214e-07,
      "loss": 0.0767,
      "num_tokens": 292619214.0,
      "reward": 0.505208358168602,
      "reward_std": 0.221573606133461,
      "rewards/accuracy_reward": 0.15624999720603228,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 1701
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.2292175292969,
      "epoch": 0.6536725876140183,
      "grad_norm": 0.236328125,
      "learning_rate": 3.9055805692124475e-07,
      "loss": 0.137,
      "num_tokens": 292796508.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.2282719984650612,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1702
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.5312805175781,
      "epoch": 0.6540566490638502,
      "grad_norm": 0.2412109375,
      "learning_rate": 3.8999372436879976e-07,
      "loss": 0.1174,
      "num_tokens": 292958575.0,
      "reward": 0.4765625223517418,
      "reward_std": 0.22694627195596695,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958432674408,
      "step": 1703
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1556.1146545410156,
      "epoch": 0.6544407105136822,
      "grad_norm": 0.259765625,
      "learning_rate": 3.894296797304425e-07,
      "loss": 0.0638,
      "num_tokens": 293121322.0,
      "reward": 0.518229179084301,
      "reward_std": 0.2102501578629017,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1704
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.0937805175781,
      "epoch": 0.6548247719635142,
      "grad_norm": 0.232421875,
      "learning_rate": 3.888659240211103e-07,
      "loss": 0.128,
      "num_tokens": 293288323.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.14845590852200985,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1705
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.3854675292969,
      "epoch": 0.6552088334133461,
      "grad_norm": 0.2255859375,
      "learning_rate": 3.883024582552212e-07,
      "loss": 0.0966,
      "num_tokens": 293466208.0,
      "reward": 0.557291679084301,
      "reward_std": 0.2665612231940031,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1706
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.8437805175781,
      "epoch": 0.6555928948631781,
      "grad_norm": 0.232421875,
      "learning_rate": 3.8773928344667107e-07,
      "loss": 0.0835,
      "num_tokens": 293631593.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.25409120600670576,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1707
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1796.0937805175781,
      "epoch": 0.6559769563130101,
      "grad_norm": 0.1767578125,
      "learning_rate": 3.8717640060883274e-07,
      "loss": 0.0907,
      "num_tokens": 293817466.0,
      "reward": 0.3906250074505806,
      "reward_std": 0.1500572357326746,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1708
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1824.8125305175781,
      "epoch": 0.6563610177628421,
      "grad_norm": 0.21484375,
      "learning_rate": 3.866138107545528e-07,
      "loss": 0.0766,
      "num_tokens": 294006400.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.24150746688246727,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1709
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.0729370117188,
      "epoch": 0.656745079212674,
      "grad_norm": 0.251953125,
      "learning_rate": 3.8605151489615184e-07,
      "loss": 0.116,
      "num_tokens": 294182991.0,
      "reward": 0.494791679084301,
      "reward_std": 0.2144372072070837,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 1710
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1785.2292175292969,
      "epoch": 0.657129140662506,
      "grad_norm": 0.2109375,
      "learning_rate": 3.8548951404542016e-07,
      "loss": 0.0459,
      "num_tokens": 294372293.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.19184456719085574,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1711
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.7396240234375,
      "epoch": 0.657513202112338,
      "grad_norm": 0.224609375,
      "learning_rate": 3.849278092136187e-07,
      "loss": 0.0976,
      "num_tokens": 294548764.0,
      "reward": 0.4609375,
      "reward_std": 0.20922628045082092,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 1712
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.6250610351562,
      "epoch": 0.6578972635621699,
      "grad_norm": 0.1962890625,
      "learning_rate": 3.843664014114738e-07,
      "loss": 0.0871,
      "num_tokens": 294721392.0,
      "reward": 0.486979179084301,
      "reward_std": 0.25846922397613525,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1713
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.0729675292969,
      "epoch": 0.6582813250120019,
      "grad_norm": 0.20703125,
      "learning_rate": 3.838052916491793e-07,
      "loss": 0.0837,
      "num_tokens": 294891599.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.21713414415717125,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1714
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.8021240234375,
      "epoch": 0.6586653864618339,
      "grad_norm": 0.251953125,
      "learning_rate": 3.832444809363916e-07,
      "loss": 0.1139,
      "num_tokens": 295067436.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.20489371195435524,
      "rewards/accuracy_reward": 0.0677083320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 1715
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1823.1979675292969,
      "epoch": 0.6590494479116659,
      "grad_norm": 0.1640625,
      "learning_rate": 3.826839702822292e-07,
      "loss": 0.0788,
      "num_tokens": 295257215.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.18230718187987804,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666716337204,
      "step": 1716
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.8334045410156,
      "epoch": 0.6594335093614978,
      "grad_norm": 0.1884765625,
      "learning_rate": 3.8212376069527097e-07,
      "loss": 0.1326,
      "num_tokens": 295434959.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.24330223724246025,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1717
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1802.7604370117188,
      "epoch": 0.6598175708113299,
      "grad_norm": 0.177734375,
      "learning_rate": 3.8156385318355356e-07,
      "loss": 0.082,
      "num_tokens": 295621400.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.2131695505231619,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1718
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1471.8958740234375,
      "epoch": 0.6602016322611618,
      "grad_norm": 0.30859375,
      "learning_rate": 3.810042487545705e-07,
      "loss": 0.0845,
      "num_tokens": 295777238.0,
      "reward": 0.5104166865348816,
      "reward_std": 0.21459411829710007,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1719
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.6042175292969,
      "epoch": 0.6605856937109937,
      "grad_norm": 0.2119140625,
      "learning_rate": 3.8044494841526975e-07,
      "loss": 0.1256,
      "num_tokens": 295958216.0,
      "reward": 0.440104179084301,
      "reward_std": 0.19228601269423962,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1720
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.5313110351562,
      "epoch": 0.6609697551608257,
      "grad_norm": 0.22265625,
      "learning_rate": 3.7988595317205206e-07,
      "loss": 0.1397,
      "num_tokens": 296125443.0,
      "reward": 0.5911458432674408,
      "reward_std": 0.3287460319697857,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291641831398,
      "step": 1721
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1670.6771240234375,
      "epoch": 0.6613538166106577,
      "grad_norm": 0.2255859375,
      "learning_rate": 3.793272640307693e-07,
      "loss": 0.1171,
      "num_tokens": 296301396.0,
      "reward": 0.526041679084301,
      "reward_std": 0.24637784622609615,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1722
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1491.8750610351562,
      "epoch": 0.6617378780604897,
      "grad_norm": 0.21484375,
      "learning_rate": 3.7876888199672263e-07,
      "loss": 0.118,
      "num_tokens": 296456488.0,
      "reward": 0.479166679084301,
      "reward_std": 0.20275012776255608,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1723
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.8646240234375,
      "epoch": 0.6621219395103216,
      "grad_norm": 0.2255859375,
      "learning_rate": 3.782108080746602e-07,
      "loss": 0.0666,
      "num_tokens": 296616363.0,
      "reward": 0.533854179084301,
      "reward_std": 0.25101388245821,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.440104179084301,
      "step": 1724
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1809.7813110351562,
      "epoch": 0.6625060009601537,
      "grad_norm": 0.1923828125,
      "learning_rate": 3.7765304326877634e-07,
      "loss": 0.0633,
      "num_tokens": 296805190.0,
      "reward": 0.3567708358168602,
      "reward_std": 0.09977966919541359,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1725
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1707.3229675292969,
      "epoch": 0.6628900624099856,
      "grad_norm": 0.2197265625,
      "learning_rate": 3.7709558858270863e-07,
      "loss": 0.0779,
      "num_tokens": 296981309.0,
      "reward": 0.458333358168602,
      "reward_std": 0.18524127081036568,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 1726
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1553.5417175292969,
      "epoch": 0.6632741238598175,
      "grad_norm": 0.232421875,
      "learning_rate": 3.765384450195369e-07,
      "loss": 0.1233,
      "num_tokens": 297143817.0,
      "reward": 0.489583358168602,
      "reward_std": 0.23642883077263832,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 1727
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.8229370117188,
      "epoch": 0.6636581853096496,
      "grad_norm": 0.22265625,
      "learning_rate": 3.75981613581781e-07,
      "loss": 0.086,
      "num_tokens": 297304528.0,
      "reward": 0.5364583432674408,
      "reward_std": 0.2588433865457773,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1728
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1366.0000305175781,
      "epoch": 0.6640422467594815,
      "grad_norm": 0.255859375,
      "learning_rate": 3.7542509527139943e-07,
      "loss": 0.1297,
      "num_tokens": 297448776.0,
      "reward": 0.5781250223517418,
      "reward_std": 0.22711073607206345,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583358168602,
      "step": 1729
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.3645935058594,
      "epoch": 0.6644263082093135,
      "grad_norm": 0.201171875,
      "learning_rate": 3.748688910897868e-07,
      "loss": 0.0863,
      "num_tokens": 297617835.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.16577523155137897,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 1730
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.1667175292969,
      "epoch": 0.6648103696591454,
      "grad_norm": 0.2109375,
      "learning_rate": 3.7431300203777296e-07,
      "loss": 0.0871,
      "num_tokens": 297792139.0,
      "reward": 0.526041679084301,
      "reward_std": 0.249834175221622,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1731
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.260482788086,
      "epoch": 0.6651944311089775,
      "grad_norm": 0.21484375,
      "learning_rate": 3.7375742911562034e-07,
      "loss": 0.1012,
      "num_tokens": 297954180.0,
      "reward": 0.5755208432674408,
      "reward_std": 0.24271745514124632,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1732
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1673.7916870117188,
      "epoch": 0.6655784925588094,
      "grad_norm": 0.2265625,
      "learning_rate": 3.732021733230226e-07,
      "loss": 0.0878,
      "num_tokens": 298130784.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.22703188098967075,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 1733
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.0208435058594,
      "epoch": 0.6659625540086413,
      "grad_norm": 0.1806640625,
      "learning_rate": 3.7264723565910314e-07,
      "loss": 0.0667,
      "num_tokens": 298295418.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.1884591169655323,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1734
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.5312805175781,
      "epoch": 0.6663466154584734,
      "grad_norm": 0.23046875,
      "learning_rate": 3.7209261712241226e-07,
      "loss": 0.1192,
      "num_tokens": 298464893.0,
      "reward": 0.4192708507180214,
      "reward_std": 0.17710472457110882,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 1735
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.7500610351562,
      "epoch": 0.6667306769083053,
      "grad_norm": 0.1826171875,
      "learning_rate": 3.7153831871092665e-07,
      "loss": 0.0945,
      "num_tokens": 298642293.0,
      "reward": 0.5208333358168602,
      "reward_std": 0.2160467803478241,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 1736
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.0313110351562,
      "epoch": 0.6671147383581373,
      "grad_norm": 0.28125,
      "learning_rate": 3.7098434142204637e-07,
      "loss": 0.1443,
      "num_tokens": 298798672.0,
      "reward": 0.5703125149011612,
      "reward_std": 0.254154734313488,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1737
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1473.4063110351562,
      "epoch": 0.6674987998079692,
      "grad_norm": 0.2255859375,
      "learning_rate": 3.7043068625259445e-07,
      "loss": 0.1039,
      "num_tokens": 298954463.0,
      "reward": 0.625,
      "reward_std": 0.2366391457617283,
      "rewards/accuracy_reward": 0.16666666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 1738
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.0000610351562,
      "epoch": 0.6678828612578013,
      "grad_norm": 0.203125,
      "learning_rate": 3.6987735419881336e-07,
      "loss": 0.1184,
      "num_tokens": 299131647.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.29086272418498993,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 1739
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1722.1354675292969,
      "epoch": 0.6682669227076332,
      "grad_norm": 0.2392578125,
      "learning_rate": 3.69324346256365e-07,
      "loss": 0.1222,
      "num_tokens": 299314548.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.24453927390277386,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1740
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1441.2292175292969,
      "epoch": 0.6686509841574652,
      "grad_norm": 0.193359375,
      "learning_rate": 3.687716634203275e-07,
      "loss": 0.1018,
      "num_tokens": 299462970.0,
      "reward": 0.6041666865348816,
      "reward_std": 0.24444082379341125,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 1741
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1845.3437805175781,
      "epoch": 0.6690350456072972,
      "grad_norm": 0.2001953125,
      "learning_rate": 3.682193066851942e-07,
      "loss": 0.0843,
      "num_tokens": 299654403.0,
      "reward": 0.3567708432674408,
      "reward_std": 0.10415726900100708,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 1742
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.9167175292969,
      "epoch": 0.6694191070571291,
      "grad_norm": 0.25390625,
      "learning_rate": 3.6766727704487165e-07,
      "loss": 0.116,
      "num_tokens": 299822283.0,
      "reward": 0.466145858168602,
      "reward_std": 0.1802374515682459,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140624925494194,
      "step": 1743
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1455.7500305175781,
      "epoch": 0.6698031685069611,
      "grad_norm": 0.2138671875,
      "learning_rate": 3.671155754926779e-07,
      "loss": 0.1021,
      "num_tokens": 299975075.0,
      "reward": 0.5364583507180214,
      "reward_std": 0.23288635909557343,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 1744
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.8750305175781,
      "epoch": 0.6701872299567931,
      "grad_norm": 0.25390625,
      "learning_rate": 3.665642030213406e-07,
      "loss": 0.1074,
      "num_tokens": 300136727.0,
      "reward": 0.5494792014360428,
      "reward_std": 0.2082697544246912,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1745
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.9166870117188,
      "epoch": 0.6705712914066251,
      "grad_norm": 0.203125,
      "learning_rate": 3.66013160622995e-07,
      "loss": 0.0839,
      "num_tokens": 300312975.0,
      "reward": 0.3750000074505806,
      "reward_std": 0.1384136280976236,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 1746
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.3333740234375,
      "epoch": 0.670955352856457,
      "grad_norm": 0.1923828125,
      "learning_rate": 3.65462449289183e-07,
      "loss": 0.0897,
      "num_tokens": 300497167.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.18351702857762575,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1747
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.0104675292969,
      "epoch": 0.671339414306289,
      "grad_norm": 0.2490234375,
      "learning_rate": 3.649120700108501e-07,
      "loss": 0.1166,
      "num_tokens": 300661288.0,
      "reward": 0.567708358168602,
      "reward_std": 0.31937991827726364,
      "rewards/accuracy_reward": 0.14583333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1748
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.7292175292969,
      "epoch": 0.671723475756121,
      "grad_norm": 0.19921875,
      "learning_rate": 3.643620237783448e-07,
      "loss": 0.1249,
      "num_tokens": 300835374.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.28194764256477356,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083432674408,
      "step": 1749
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1322.885498046875,
      "epoch": 0.672107537205953,
      "grad_norm": 0.302734375,
      "learning_rate": 3.63812311581416e-07,
      "loss": 0.0801,
      "num_tokens": 300975787.0,
      "reward": 0.5859375149011612,
      "reward_std": 0.27406129613518715,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5234375,
      "step": 1750
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.3958740234375,
      "epoch": 0.6724915986557849,
      "grad_norm": 0.1884765625,
      "learning_rate": 3.63262934409212e-07,
      "loss": 0.0256,
      "num_tokens": 301162305.0,
      "reward": 0.479166679084301,
      "reward_std": 0.2071267617866397,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1751
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1491.6041870117188,
      "epoch": 0.672875660105617,
      "grad_norm": 0.236328125,
      "learning_rate": 3.6271389325027747e-07,
      "loss": 0.1362,
      "num_tokens": 301321131.0,
      "reward": 0.5338541865348816,
      "reward_std": 0.31443943828344345,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 1752
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1416.8125610351562,
      "epoch": 0.6732597215554489,
      "grad_norm": 0.283203125,
      "learning_rate": 3.621651890925536e-07,
      "loss": 0.0708,
      "num_tokens": 301469521.0,
      "reward": 0.5963541865348816,
      "reward_std": 0.2515166364610195,
      "rewards/accuracy_reward": 0.15104166883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541641831398,
      "step": 1753
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.2917175292969,
      "epoch": 0.6736437830052808,
      "grad_norm": 0.2255859375,
      "learning_rate": 3.616168229233737e-07,
      "loss": 0.0813,
      "num_tokens": 301641717.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.1829132717102766,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1754
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.7291870117188,
      "epoch": 0.6740278444551128,
      "grad_norm": 0.2314453125,
      "learning_rate": 3.6106879572946436e-07,
      "loss": 0.1575,
      "num_tokens": 301811059.0,
      "reward": 0.4427083358168602,
      "reward_std": 0.192892674356699,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 1755
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1831.5625305175781,
      "epoch": 0.6744119059049448,
      "grad_norm": 0.2021484375,
      "learning_rate": 3.605211084969414e-07,
      "loss": 0.1106,
      "num_tokens": 302003593.0,
      "reward": 0.3515625149011612,
      "reward_std": 0.14072956843301654,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515624925494194,
      "step": 1756
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.7604522705078,
      "epoch": 0.6747959673547768,
      "grad_norm": 0.23828125,
      "learning_rate": 3.5997376221130915e-07,
      "loss": 0.0995,
      "num_tokens": 302169922.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.19983750581741333,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1757
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.1146545410156,
      "epoch": 0.6751800288046087,
      "grad_norm": 0.203125,
      "learning_rate": 3.5942675785745804e-07,
      "loss": 0.1039,
      "num_tokens": 302343053.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.15205310843884945,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1758
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1448.3750305175781,
      "epoch": 0.6755640902544408,
      "grad_norm": 0.2578125,
      "learning_rate": 3.588800964196639e-07,
      "loss": 0.1957,
      "num_tokens": 302497233.0,
      "reward": 0.6692708507180214,
      "reward_std": 0.31440648436546326,
      "rewards/accuracy_reward": 0.1979166716337204,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541716337204,
      "step": 1759
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1467.8646240234375,
      "epoch": 0.6759481517042727,
      "grad_norm": 0.2353515625,
      "learning_rate": 3.5833377888158554e-07,
      "loss": 0.062,
      "num_tokens": 302652156.0,
      "reward": 0.6692708507180214,
      "reward_std": 0.26844339817762375,
      "rewards/accuracy_reward": 0.20833333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375074505806,
      "step": 1760
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.4375305175781,
      "epoch": 0.6763322131541046,
      "grad_norm": 0.2236328125,
      "learning_rate": 3.5778780622626194e-07,
      "loss": 0.1354,
      "num_tokens": 302822398.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.1826671976596117,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1761
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.5104675292969,
      "epoch": 0.6767162746039367,
      "grad_norm": 0.21875,
      "learning_rate": 3.5724217943611255e-07,
      "loss": 0.1228,
      "num_tokens": 302982135.0,
      "reward": 0.5130208507180214,
      "reward_std": 0.2473776899278164,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4817708358168602,
      "step": 1762
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.3854675292969,
      "epoch": 0.6771003360537686,
      "grad_norm": 0.2001953125,
      "learning_rate": 3.5669689949293404e-07,
      "loss": 0.1438,
      "num_tokens": 303165964.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.2017520796507597,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 1763
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.7396240234375,
      "epoch": 0.6774843975036006,
      "grad_norm": 0.2373046875,
      "learning_rate": 3.561519673778993e-07,
      "loss": 0.0367,
      "num_tokens": 303345875.0,
      "reward": 0.4140625,
      "reward_std": 0.14422015473246574,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1764
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.7813110351562,
      "epoch": 0.6778684589534325,
      "grad_norm": 0.2421875,
      "learning_rate": 3.5560738407155457e-07,
      "loss": 0.0854,
      "num_tokens": 303519094.0,
      "reward": 0.518229179084301,
      "reward_std": 0.2906688563525677,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1765
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.4063110351562,
      "epoch": 0.6782525204032646,
      "grad_norm": 0.265625,
      "learning_rate": 3.550631505538193e-07,
      "loss": 0.0955,
      "num_tokens": 303687245.0,
      "reward": 0.450520858168602,
      "reward_std": 0.17868239618837833,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 1766
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.5208740234375,
      "epoch": 0.6786365818530965,
      "grad_norm": 0.1904296875,
      "learning_rate": 3.54519267803983e-07,
      "loss": 0.0223,
      "num_tokens": 303868463.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.1763669503852725,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1767
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.1250305175781,
      "epoch": 0.6790206433029284,
      "grad_norm": 0.1748046875,
      "learning_rate": 3.539757368007046e-07,
      "loss": 0.0686,
      "num_tokens": 304045515.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.1753207053989172,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1768
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.9792175292969,
      "epoch": 0.6794047047527605,
      "grad_norm": 0.2041015625,
      "learning_rate": 3.5343255852200906e-07,
      "loss": 0.1123,
      "num_tokens": 304227681.0,
      "reward": 0.3750000074505806,
      "reward_std": 0.11345137841999531,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1769
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.5416870117188,
      "epoch": 0.6797887662025924,
      "grad_norm": 0.2373046875,
      "learning_rate": 3.528897339452878e-07,
      "loss": 0.1203,
      "num_tokens": 304387357.0,
      "reward": 0.6302083432674408,
      "reward_std": 0.22957686707377434,
      "rewards/accuracy_reward": 0.2083333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1770
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.1458740234375,
      "epoch": 0.6801728276524244,
      "grad_norm": 0.2294921875,
      "learning_rate": 3.5234726404729486e-07,
      "loss": 0.1277,
      "num_tokens": 304555667.0,
      "reward": 0.4947916716337204,
      "reward_std": 0.28148072957992554,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1771
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.1979675292969,
      "epoch": 0.6805568891022563,
      "grad_norm": 0.232421875,
      "learning_rate": 3.518051498041471e-07,
      "loss": 0.1462,
      "num_tokens": 304729254.0,
      "reward": 0.471354179084301,
      "reward_std": 0.2097102366387844,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1772
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.4375305175781,
      "epoch": 0.6809409505520884,
      "grad_norm": 0.2041015625,
      "learning_rate": 3.512633921913201e-07,
      "loss": 0.0706,
      "num_tokens": 304920120.0,
      "reward": 0.3880208507180214,
      "reward_std": 0.146726593375206,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 1773
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.6458740234375,
      "epoch": 0.6813250120019203,
      "grad_norm": 0.23828125,
      "learning_rate": 3.5072199218364874e-07,
      "loss": 0.0716,
      "num_tokens": 305082478.0,
      "reward": 0.5703125149011612,
      "reward_std": 0.21707365615293384,
      "rewards/accuracy_reward": 0.15625000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 1774
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.4896240234375,
      "epoch": 0.6817090734517522,
      "grad_norm": 0.205078125,
      "learning_rate": 3.5018095075532414e-07,
      "loss": 0.0839,
      "num_tokens": 305255573.0,
      "reward": 0.440104179084301,
      "reward_std": 0.19830374978482723,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1775
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.8229675292969,
      "epoch": 0.6820931349015843,
      "grad_norm": 0.1982421875,
      "learning_rate": 3.4964026887989234e-07,
      "loss": 0.1155,
      "num_tokens": 305420652.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.2614674009382725,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 1776
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.7500305175781,
      "epoch": 0.6824771963514162,
      "grad_norm": 0.236328125,
      "learning_rate": 3.4909994753025204e-07,
      "loss": 0.1101,
      "num_tokens": 305589244.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.20445317961275578,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1777
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.1979675292969,
      "epoch": 0.6828612578012482,
      "grad_norm": 0.2041015625,
      "learning_rate": 3.485599876786532e-07,
      "loss": 0.1152,
      "num_tokens": 305761423.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.18837566301226616,
      "rewards/accuracy_reward": 0.0572916679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1778
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.2916870117188,
      "epoch": 0.6832453192510801,
      "grad_norm": 0.2041015625,
      "learning_rate": 3.4802039029669605e-07,
      "loss": 0.0714,
      "num_tokens": 305930547.0,
      "reward": 0.4583333358168602,
      "reward_std": 0.16059935092926025,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1779
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.8229675292969,
      "epoch": 0.6836293807009122,
      "grad_norm": 0.21484375,
      "learning_rate": 3.474811563553275e-07,
      "loss": 0.0902,
      "num_tokens": 306107954.0,
      "reward": 0.5729166939854622,
      "reward_std": 0.3257361799478531,
      "rewards/accuracy_reward": 0.1770833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333432674408,
      "step": 1780
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1508.2813110351562,
      "epoch": 0.6840134421507441,
      "grad_norm": 0.2255859375,
      "learning_rate": 3.4694228682484147e-07,
      "loss": 0.0844,
      "num_tokens": 306266949.0,
      "reward": 0.6380208432674408,
      "reward_std": 0.28080678544938564,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 1781
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1318.062515258789,
      "epoch": 0.684397503600576,
      "grad_norm": 0.279296875,
      "learning_rate": 3.4640378267487523e-07,
      "loss": 0.0604,
      "num_tokens": 306406403.0,
      "reward": 0.6067708432674408,
      "reward_std": 0.3152304142713547,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4817708358168602,
      "step": 1782
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.4896240234375,
      "epoch": 0.6847815650504081,
      "grad_norm": 0.1904296875,
      "learning_rate": 3.4586564487440994e-07,
      "loss": 0.1193,
      "num_tokens": 306577874.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.13044471200555563,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1783
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.7500610351562,
      "epoch": 0.68516562650024,
      "grad_norm": 0.1806640625,
      "learning_rate": 3.453278743917657e-07,
      "loss": 0.0812,
      "num_tokens": 306752314.0,
      "reward": 0.416666679084301,
      "reward_std": 0.15714666713029146,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1784
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1499.6458740234375,
      "epoch": 0.685549687950072,
      "grad_norm": 0.2333984375,
      "learning_rate": 3.4479047219460336e-07,
      "loss": 0.0479,
      "num_tokens": 306910464.0,
      "reward": 0.5703125074505806,
      "reward_std": 0.2086882870644331,
      "rewards/accuracy_reward": 0.14583333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791567325592,
      "step": 1785
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.5625610351562,
      "epoch": 0.685933749399904,
      "grad_norm": 0.2158203125,
      "learning_rate": 3.4425343924992005e-07,
      "loss": 0.0948,
      "num_tokens": 307094118.0,
      "reward": 0.447916679084301,
      "reward_std": 0.19612015783786774,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 1786
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1447.7708435058594,
      "epoch": 0.686317810849736,
      "grad_norm": 0.25,
      "learning_rate": 3.437167765240493e-07,
      "loss": 0.0651,
      "num_tokens": 307249112.0,
      "reward": 0.583333358168602,
      "reward_std": 0.2154905218631029,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 1787
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.03125,
      "epoch": 0.6867018722995679,
      "grad_norm": 0.1982421875,
      "learning_rate": 3.431804849826571e-07,
      "loss": 0.0591,
      "num_tokens": 307431515.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.19936397299170494,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1788
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1722.3646545410156,
      "epoch": 0.6870859337493999,
      "grad_norm": 0.216796875,
      "learning_rate": 3.4264456559074306e-07,
      "loss": 0.0921,
      "num_tokens": 307611782.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.19736266136169434,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 1789
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.6146240234375,
      "epoch": 0.6874699951992319,
      "grad_norm": 0.2451171875,
      "learning_rate": 3.4210901931263645e-07,
      "loss": 0.0955,
      "num_tokens": 307777353.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.23029962554574013,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875074505806,
      "step": 1790
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.9583740234375,
      "epoch": 0.6878540566490638,
      "grad_norm": 0.2275390625,
      "learning_rate": 3.415738471119952e-07,
      "loss": 0.1034,
      "num_tokens": 307943725.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.3094714730978012,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1791
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.0208740234375,
      "epoch": 0.6882381180988958,
      "grad_norm": 0.2333984375,
      "learning_rate": 3.410390499518041e-07,
      "loss": 0.1308,
      "num_tokens": 308112903.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.2799031138420105,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1792
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1601.6042175292969,
      "epoch": 0.6886221795487278,
      "grad_norm": 0.205078125,
      "learning_rate": 3.405046287943728e-07,
      "loss": 0.1065,
      "num_tokens": 308280697.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.1949633057229221,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140624925494194,
      "step": 1793
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.1250305175781,
      "epoch": 0.6890062409985598,
      "grad_norm": 0.2177734375,
      "learning_rate": 3.399705846013352e-07,
      "loss": 0.1112,
      "num_tokens": 308450213.0,
      "reward": 0.4479166716337204,
      "reward_std": 0.19606789015233517,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1794
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.4271240234375,
      "epoch": 0.6893903024483917,
      "grad_norm": 0.23046875,
      "learning_rate": 3.394369183336462e-07,
      "loss": 0.0917,
      "num_tokens": 308616702.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.22160671092569828,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 1795
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.1875610351562,
      "epoch": 0.6897743638982237,
      "grad_norm": 0.2353515625,
      "learning_rate": 3.3890363095158094e-07,
      "loss": 0.1964,
      "num_tokens": 308788256.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.20328206941485405,
      "rewards/accuracy_reward": 0.015625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1796
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1858.8854675292969,
      "epoch": 0.6901584253480557,
      "grad_norm": 0.2041015625,
      "learning_rate": 3.383707234147324e-07,
      "loss": 0.107,
      "num_tokens": 308982949.0,
      "reward": 0.408854179084301,
      "reward_std": 0.18716737441718578,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 1797
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.5833435058594,
      "epoch": 0.6905424867978877,
      "grad_norm": 0.23828125,
      "learning_rate": 3.3783819668201095e-07,
      "loss": 0.1011,
      "num_tokens": 309153805.0,
      "reward": 0.5494791865348816,
      "reward_std": 0.2878502905368805,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 1798
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.1354675292969,
      "epoch": 0.6909265482477196,
      "grad_norm": 0.197265625,
      "learning_rate": 3.3730605171164057e-07,
      "loss": 0.0964,
      "num_tokens": 309332858.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.1772642806172371,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1799
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.0625610351562,
      "epoch": 0.6913106096975516,
      "grad_norm": 0.2197265625,
      "learning_rate": 3.367742894611593e-07,
      "loss": 0.1292,
      "num_tokens": 309501656.0,
      "reward": 0.5026041716337204,
      "reward_std": 0.2444329559803009,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1800
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.0833740234375,
      "epoch": 0.6916946711473836,
      "grad_norm": 0.23046875,
      "learning_rate": 3.3624291088741595e-07,
      "loss": 0.1091,
      "num_tokens": 309661192.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.24152443557977676,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1801
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.9479675292969,
      "epoch": 0.6920787325972155,
      "grad_norm": 0.255859375,
      "learning_rate": 3.3571191694656953e-07,
      "loss": 0.0821,
      "num_tokens": 309815171.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.1964675784111023,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 1802
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.541748046875,
      "epoch": 0.6924627940470476,
      "grad_norm": 0.177734375,
      "learning_rate": 3.351813085940858e-07,
      "loss": 0.0467,
      "num_tokens": 309990031.0,
      "reward": 0.4635416716337204,
      "reward_std": 0.2005254253745079,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 1803
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.8333740234375,
      "epoch": 0.6928468554968795,
      "grad_norm": 0.2421875,
      "learning_rate": 3.34651086784738e-07,
      "loss": 0.0972,
      "num_tokens": 310158679.0,
      "reward": 0.5000000074505806,
      "reward_std": 0.26055463030934334,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 1804
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.5000610351562,
      "epoch": 0.6932309169467115,
      "grad_norm": 0.181640625,
      "learning_rate": 3.3412125247260316e-07,
      "loss": 0.0905,
      "num_tokens": 310330783.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.18117788899689913,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 1805
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1848.4062805175781,
      "epoch": 0.6936149783965434,
      "grad_norm": 0.1767578125,
      "learning_rate": 3.3359180661106116e-07,
      "loss": 0.0656,
      "num_tokens": 310525566.0,
      "reward": 0.434895858168602,
      "reward_std": 0.1829746845178306,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291567325592,
      "step": 1806
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.9687805175781,
      "epoch": 0.6939990398463755,
      "grad_norm": 0.19921875,
      "learning_rate": 3.3306275015279285e-07,
      "loss": 0.1098,
      "num_tokens": 310712107.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.19502881169319153,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3750000074505806,
      "step": 1807
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1293.3333892822266,
      "epoch": 0.6943831012962074,
      "grad_norm": 0.3125,
      "learning_rate": 3.3253408404977824e-07,
      "loss": 0.0458,
      "num_tokens": 310848515.0,
      "reward": 0.5338541865348816,
      "reward_std": 0.18309897556900978,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5026041641831398,
      "step": 1808
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.25,
      "epoch": 0.6947671627460393,
      "grad_norm": 0.2177734375,
      "learning_rate": 3.3200580925329547e-07,
      "loss": 0.0701,
      "num_tokens": 311017947.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.14299216121435165,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 1809
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.3646240234375,
      "epoch": 0.6951512241958714,
      "grad_norm": 0.2109375,
      "learning_rate": 3.3147792671391817e-07,
      "loss": 0.1184,
      "num_tokens": 311197598.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.1571465078741312,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 1810
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1476.75,
      "epoch": 0.6955352856457033,
      "grad_norm": 0.236328125,
      "learning_rate": 3.30950437381514e-07,
      "loss": 0.0874,
      "num_tokens": 311352878.0,
      "reward": 0.5911458432674408,
      "reward_std": 0.23600266827270389,
      "rewards/accuracy_reward": 0.1458333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1811
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1859.9583740234375,
      "epoch": 0.6959193470955353,
      "grad_norm": 0.1787109375,
      "learning_rate": 3.3042334220524335e-07,
      "loss": 0.1052,
      "num_tokens": 311544978.0,
      "reward": 0.4088541716337204,
      "reward_std": 0.1970070842653513,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1812
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.6875305175781,
      "epoch": 0.6963034085453672,
      "grad_norm": 0.1904296875,
      "learning_rate": 3.298966421335575e-07,
      "loss": 0.1136,
      "num_tokens": 311711172.0,
      "reward": 0.4401041865348816,
      "reward_std": 0.1478651212528348,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1813
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1267.4896240234375,
      "epoch": 0.6966874699951993,
      "grad_norm": 0.279296875,
      "learning_rate": 3.2937033811419645e-07,
      "loss": 0.1066,
      "num_tokens": 311845963.0,
      "reward": 0.6328125223517418,
      "reward_std": 0.275703776627779,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 1814
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1596.8125610351562,
      "epoch": 0.6970715314450312,
      "grad_norm": 0.2421875,
      "learning_rate": 3.288444310941878e-07,
      "loss": 0.1067,
      "num_tokens": 312018377.0,
      "reward": 0.4817708358168602,
      "reward_std": 0.18719684332609177,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1815
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.3229370117188,
      "epoch": 0.6974555928948631,
      "grad_norm": 0.2197265625,
      "learning_rate": 3.2831892201984437e-07,
      "loss": 0.0748,
      "num_tokens": 312193984.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.1413288749754429,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 1816
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1857.0625305175781,
      "epoch": 0.6978396543446952,
      "grad_norm": 0.1591796875,
      "learning_rate": 3.2779381183676363e-07,
      "loss": 0.098,
      "num_tokens": 312386222.0,
      "reward": 0.369791679084301,
      "reward_std": 0.15117893368005753,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.328125,
      "step": 1817
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.5208740234375,
      "epoch": 0.6982237157945271,
      "grad_norm": 0.1982421875,
      "learning_rate": 3.2726910148982466e-07,
      "loss": 0.1191,
      "num_tokens": 312564272.0,
      "reward": 0.627604179084301,
      "reward_std": 0.3374447450041771,
      "rewards/accuracy_reward": 0.2083333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1818
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.072998046875,
      "epoch": 0.6986077772443591,
      "grad_norm": 0.201171875,
      "learning_rate": 3.267447919231873e-07,
      "loss": 0.1616,
      "num_tokens": 312742959.0,
      "reward": 0.520833358168602,
      "reward_std": 0.26236492395401,
      "rewards/accuracy_reward": 0.1302083358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1819
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1440.3334045410156,
      "epoch": 0.6989918386941911,
      "grad_norm": 0.24609375,
      "learning_rate": 3.2622088408029036e-07,
      "loss": 0.1223,
      "num_tokens": 312892687.0,
      "reward": 0.5520833432674408,
      "reward_std": 0.2759733870625496,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687499925494194,
      "step": 1820
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1347.3854675292969,
      "epoch": 0.6993759001440231,
      "grad_norm": 0.25,
      "learning_rate": 3.256973789038496e-07,
      "loss": 0.064,
      "num_tokens": 313035356.0,
      "reward": 0.7421875149011612,
      "reward_std": 0.3265930637717247,
      "rewards/accuracy_reward": 0.2604166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4817708283662796,
      "step": 1821
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.7916870117188,
      "epoch": 0.699759961593855,
      "grad_norm": 0.201171875,
      "learning_rate": 3.2517427733585634e-07,
      "loss": 0.1488,
      "num_tokens": 313223968.0,
      "reward": 0.3645833432674408,
      "reward_std": 0.14590240642428398,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 1822
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1493.0937805175781,
      "epoch": 0.700144023043687,
      "grad_norm": 0.212890625,
      "learning_rate": 3.246515803175753e-07,
      "loss": 0.0651,
      "num_tokens": 313380737.0,
      "reward": 0.5234375223517418,
      "reward_std": 0.1610101405531168,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041567325592,
      "step": 1823
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.3646240234375,
      "epoch": 0.700528084493519,
      "grad_norm": 0.21484375,
      "learning_rate": 3.241292887895442e-07,
      "loss": 0.0997,
      "num_tokens": 313546612.0,
      "reward": 0.666666679084301,
      "reward_std": 0.2564549222588539,
      "rewards/accuracy_reward": 0.21875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 1824
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.9062805175781,
      "epoch": 0.7009121459433509,
      "grad_norm": 0.205078125,
      "learning_rate": 3.236074036915699e-07,
      "loss": 0.0801,
      "num_tokens": 313722043.0,
      "reward": 0.4557291716337204,
      "reward_std": 0.19483141042292118,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 1825
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1488.7708740234375,
      "epoch": 0.7012962073931829,
      "grad_norm": 0.2431640625,
      "learning_rate": 3.2308592596272876e-07,
      "loss": 0.1475,
      "num_tokens": 313881253.0,
      "reward": 0.622395858168602,
      "reward_std": 0.28834143653512,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291641831398,
      "step": 1826
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.3750305175781,
      "epoch": 0.7016802688430149,
      "grad_norm": 0.2412109375,
      "learning_rate": 3.225648565413637e-07,
      "loss": 0.0993,
      "num_tokens": 314055969.0,
      "reward": 0.471354179084301,
      "reward_std": 0.1807345012202859,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 1827
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.6354675292969,
      "epoch": 0.7020643302928469,
      "grad_norm": 0.2109375,
      "learning_rate": 3.220441963650834e-07,
      "loss": 0.0908,
      "num_tokens": 314230006.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.22161973267793655,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1828
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.9062805175781,
      "epoch": 0.7024483917426788,
      "grad_norm": 0.2314453125,
      "learning_rate": 3.2152394637075977e-07,
      "loss": 0.1013,
      "num_tokens": 314392621.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.19131004810333252,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4374999925494194,
      "step": 1829
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1305.2083435058594,
      "epoch": 0.7028324531925108,
      "grad_norm": 0.2353515625,
      "learning_rate": 3.210041074945266e-07,
      "loss": 0.0946,
      "num_tokens": 314531017.0,
      "reward": 0.7135416865348816,
      "reward_std": 0.29972394928336143,
      "rewards/accuracy_reward": 0.2291666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4843750074505806,
      "step": 1830
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.8958740234375,
      "epoch": 0.7032165146423428,
      "grad_norm": 0.373046875,
      "learning_rate": 3.2048468067177797e-07,
      "loss": 0.1445,
      "num_tokens": 314695039.0,
      "reward": 0.5000000074505806,
      "reward_std": 0.2062554992735386,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 1831
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.5104370117188,
      "epoch": 0.7036005760921747,
      "grad_norm": 0.220703125,
      "learning_rate": 3.199656668371669e-07,
      "loss": 0.1137,
      "num_tokens": 314868552.0,
      "reward": 0.497395858168602,
      "reward_std": 0.2395680844783783,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1832
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.6667175292969,
      "epoch": 0.7039846375420067,
      "grad_norm": 0.1962890625,
      "learning_rate": 3.1944706692460286e-07,
      "loss": 0.0989,
      "num_tokens": 315038024.0,
      "reward": 0.4895833432674408,
      "reward_std": 0.18514439836144447,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1833
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1597.0833740234375,
      "epoch": 0.7043686989918387,
      "grad_norm": 0.228515625,
      "learning_rate": 3.189288818672504e-07,
      "loss": 0.0387,
      "num_tokens": 315205280.0,
      "reward": 0.5286458432674408,
      "reward_std": 0.28245807997882366,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1834
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.3854675292969,
      "epoch": 0.7047527604416707,
      "grad_norm": 0.21484375,
      "learning_rate": 3.184111125975281e-07,
      "loss": 0.0852,
      "num_tokens": 315369213.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.20183612406253815,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 1835
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1678.6146240234375,
      "epoch": 0.7051368218915026,
      "grad_norm": 0.19140625,
      "learning_rate": 3.178937600471061e-07,
      "loss": 0.0849,
      "num_tokens": 315546304.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.19198534823954105,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 1836
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.6875610351562,
      "epoch": 0.7055208833413346,
      "grad_norm": 0.2119140625,
      "learning_rate": 3.1737682514690444e-07,
      "loss": 0.035,
      "num_tokens": 315717210.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.16026961570605636,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1837
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.3333740234375,
      "epoch": 0.7059049447911666,
      "grad_norm": 0.203125,
      "learning_rate": 3.1686030882709194e-07,
      "loss": 0.1171,
      "num_tokens": 315896506.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.23842930793762207,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1838
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1801.1146240234375,
      "epoch": 0.7062890062409986,
      "grad_norm": 0.17578125,
      "learning_rate": 3.163442120170844e-07,
      "loss": 0.126,
      "num_tokens": 316083605.0,
      "reward": 0.4192708507180214,
      "reward_std": 0.23656390979886055,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 1839
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.6250305175781,
      "epoch": 0.7066730676908305,
      "grad_norm": 0.1767578125,
      "learning_rate": 3.1582853564554236e-07,
      "loss": 0.0543,
      "num_tokens": 316252601.0,
      "reward": 0.4843750074505806,
      "reward_std": 0.2019559107720852,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 1840
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.9583435058594,
      "epoch": 0.7070571291406625,
      "grad_norm": 0.1396484375,
      "learning_rate": 3.153132806403701e-07,
      "loss": 0.0444,
      "num_tokens": 316436973.0,
      "reward": 0.3723958432674408,
      "reward_std": 0.12998445518314838,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458283662796,
      "step": 1841
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.4687805175781,
      "epoch": 0.7074411905904945,
      "grad_norm": 0.205078125,
      "learning_rate": 3.1479844792871325e-07,
      "loss": 0.0593,
      "num_tokens": 316607002.0,
      "reward": 0.455729179084301,
      "reward_std": 0.16921230778098106,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1842
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.4792175292969,
      "epoch": 0.7078252520403264,
      "grad_norm": 0.1904296875,
      "learning_rate": 3.1428403843695837e-07,
      "loss": 0.146,
      "num_tokens": 316787112.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.1873235534876585,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1843
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.3333740234375,
      "epoch": 0.7082093134901585,
      "grad_norm": 0.2470703125,
      "learning_rate": 3.137700530907299e-07,
      "loss": 0.1118,
      "num_tokens": 316950296.0,
      "reward": 0.5651041939854622,
      "reward_std": 0.28676314651966095,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541641831398,
      "step": 1844
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.6979675292969,
      "epoch": 0.7085933749399904,
      "grad_norm": 0.251953125,
      "learning_rate": 3.132564928148891e-07,
      "loss": 0.1693,
      "num_tokens": 317116587.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.23885615170001984,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458283662796,
      "step": 1845
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1702.7604675292969,
      "epoch": 0.7089774363898224,
      "grad_norm": 0.1923828125,
      "learning_rate": 3.127433585335324e-07,
      "loss": 0.0592,
      "num_tokens": 317294148.0,
      "reward": 0.4609375223517418,
      "reward_std": 0.2241070680320263,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1846
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1487.1875305175781,
      "epoch": 0.7093614978396543,
      "grad_norm": 0.205078125,
      "learning_rate": 3.1223065116999005e-07,
      "loss": 0.112,
      "num_tokens": 317450942.0,
      "reward": 0.5442708507180214,
      "reward_std": 0.2527505960315466,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 1847
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1839.2083435058594,
      "epoch": 0.7097455592894863,
      "grad_norm": 0.1162109375,
      "learning_rate": 3.1171837164682354e-07,
      "loss": 0.0582,
      "num_tokens": 317643866.0,
      "reward": 0.3411458432674408,
      "reward_std": 0.061024900525808334,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458432674408,
      "step": 1848
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1771.6875305175781,
      "epoch": 0.7101296207393183,
      "grad_norm": 0.1806640625,
      "learning_rate": 3.112065208858247e-07,
      "loss": 0.1055,
      "num_tokens": 317827884.0,
      "reward": 0.393229179084301,
      "reward_std": 0.17799291759729385,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1849
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.2187805175781,
      "epoch": 0.7105136821891502,
      "grad_norm": 0.189453125,
      "learning_rate": 3.1069509980801443e-07,
      "loss": 0.1098,
      "num_tokens": 318010417.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.18447720259428024,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 1850
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1707.3646240234375,
      "epoch": 0.7108977436389823,
      "grad_norm": 0.1845703125,
      "learning_rate": 3.1018410933363957e-07,
      "loss": 0.0312,
      "num_tokens": 318187932.0,
      "reward": 0.5208333358168602,
      "reward_std": 0.20229130145162344,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 1851
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.885498046875,
      "epoch": 0.7112818050888142,
      "grad_norm": 0.2177734375,
      "learning_rate": 3.096735503821726e-07,
      "loss": 0.1657,
      "num_tokens": 318360609.0,
      "reward": 0.4635416716337204,
      "reward_std": 0.22689715772867203,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 1852
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.4375610351562,
      "epoch": 0.7116658665386462,
      "grad_norm": 0.1826171875,
      "learning_rate": 3.0916342387230934e-07,
      "loss": 0.0897,
      "num_tokens": 318535787.0,
      "reward": 0.4166666716337204,
      "reward_std": 0.1540845539420843,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1853
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.3333435058594,
      "epoch": 0.7120499279884781,
      "grad_norm": 0.1669921875,
      "learning_rate": 3.086537307219679e-07,
      "loss": 0.0435,
      "num_tokens": 318725787.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.13184540532529354,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 1854
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.5104675292969,
      "epoch": 0.7124339894383102,
      "grad_norm": 0.255859375,
      "learning_rate": 3.0814447184828605e-07,
      "loss": 0.1228,
      "num_tokens": 318884540.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.17889069765806198,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1855
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.4167175292969,
      "epoch": 0.7128180508881421,
      "grad_norm": 0.220703125,
      "learning_rate": 3.076356481676206e-07,
      "loss": 0.0767,
      "num_tokens": 319051532.0,
      "reward": 0.4583333507180214,
      "reward_std": 0.17722809594124556,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666567325592,
      "step": 1856
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.5312805175781,
      "epoch": 0.713202112337974,
      "grad_norm": 0.216796875,
      "learning_rate": 3.0712726059554456e-07,
      "loss": 0.0525,
      "num_tokens": 319214031.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.16270765662193298,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453124925494194,
      "step": 1857
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1258.0313110351562,
      "epoch": 0.7135861737878061,
      "grad_norm": 0.296875,
      "learning_rate": 3.066193100468473e-07,
      "loss": 0.0853,
      "num_tokens": 319348722.0,
      "reward": 0.6354166939854622,
      "reward_std": 0.2636132054030895,
      "rewards/accuracy_reward": 0.15624999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4791666716337204,
      "step": 1858
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.1250610351562,
      "epoch": 0.713970235237638,
      "grad_norm": 0.1865234375,
      "learning_rate": 3.0611179743553083e-07,
      "loss": 0.0657,
      "num_tokens": 319509878.0,
      "reward": 0.5520833432674408,
      "reward_std": 0.2521050963550806,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 1859
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1444.3229675292969,
      "epoch": 0.71435429668747,
      "grad_norm": 0.25,
      "learning_rate": 3.056047236748097e-07,
      "loss": 0.1049,
      "num_tokens": 319661277.0,
      "reward": 0.471354179084301,
      "reward_std": 0.2156130215153098,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708432674408,
      "step": 1860
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.8021240234375,
      "epoch": 0.714738358137302,
      "grad_norm": 0.228515625,
      "learning_rate": 3.0509808967710835e-07,
      "loss": 0.1258,
      "num_tokens": 319832418.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.22898869961500168,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1861
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1777.5729675292969,
      "epoch": 0.715122419587134,
      "grad_norm": 0.2060546875,
      "learning_rate": 3.0459189635406067e-07,
      "loss": 0.0932,
      "num_tokens": 320019489.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.10433619748800993,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 1862
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.5833435058594,
      "epoch": 0.7155064810369659,
      "grad_norm": 0.1748046875,
      "learning_rate": 3.0408614461650674e-07,
      "loss": 0.1203,
      "num_tokens": 320196393.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.24689798802137375,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1863
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.1563110351562,
      "epoch": 0.7158905424867978,
      "grad_norm": 0.24609375,
      "learning_rate": 3.035808353744923e-07,
      "loss": 0.0877,
      "num_tokens": 320358864.0,
      "reward": 0.4817708358168602,
      "reward_std": 0.16738871624693274,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1864
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1439.0833740234375,
      "epoch": 0.7162746039366299,
      "grad_norm": 0.25390625,
      "learning_rate": 3.030759695372674e-07,
      "loss": 0.0601,
      "num_tokens": 320511848.0,
      "reward": 0.5416666865348816,
      "reward_std": 0.23328793048858643,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 1865
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1524.3750305175781,
      "epoch": 0.7166586653864618,
      "grad_norm": 0.2236328125,
      "learning_rate": 3.025715480132838e-07,
      "loss": 0.1556,
      "num_tokens": 320671860.0,
      "reward": 0.5885416939854622,
      "reward_std": 0.2375707682222128,
      "rewards/accuracy_reward": 0.17708333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 1866
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.5833740234375,
      "epoch": 0.7170427268362938,
      "grad_norm": 0.212890625,
      "learning_rate": 3.020675717101935e-07,
      "loss": 0.0776,
      "num_tokens": 320842780.0,
      "reward": 0.5026041716337204,
      "reward_std": 0.23631439357995987,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1867
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.5104370117188,
      "epoch": 0.7174267882861258,
      "grad_norm": 0.16796875,
      "learning_rate": 3.0156404153484763e-07,
      "loss": 0.0875,
      "num_tokens": 321022693.0,
      "reward": 0.3802083358168602,
      "reward_std": 0.09308475721627474,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1868
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1472.4479370117188,
      "epoch": 0.7178108497359578,
      "grad_norm": 0.2158203125,
      "learning_rate": 3.0106095839329487e-07,
      "loss": 0.1129,
      "num_tokens": 321177224.0,
      "reward": 0.5078125074505806,
      "reward_std": 0.21947369910776615,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 1869
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.1875305175781,
      "epoch": 0.7181949111857897,
      "grad_norm": 0.234375,
      "learning_rate": 3.005583231907792e-07,
      "loss": 0.0873,
      "num_tokens": 321340746.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.237678375095129,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 1870
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.4583740234375,
      "epoch": 0.7185789726356216,
      "grad_norm": 0.1669921875,
      "learning_rate": 3.0005613683173856e-07,
      "loss": 0.0889,
      "num_tokens": 321521478.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.18201387859880924,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 1871
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.4271545410156,
      "epoch": 0.7189630340854537,
      "grad_norm": 0.2421875,
      "learning_rate": 2.9955440021980307e-07,
      "loss": 0.1115,
      "num_tokens": 321700735.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.2125512659549713,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 1872
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.9271240234375,
      "epoch": 0.7193470955352856,
      "grad_norm": 0.2265625,
      "learning_rate": 2.990531142577942e-07,
      "loss": 0.1415,
      "num_tokens": 321884808.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.28679506480693817,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1873
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1768.7396240234375,
      "epoch": 0.7197311569851176,
      "grad_norm": 0.2216796875,
      "learning_rate": 2.98552279847722e-07,
      "loss": 0.0797,
      "num_tokens": 322070455.0,
      "reward": 0.3958333507180214,
      "reward_std": 0.14668673276901245,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 1874
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.1042175292969,
      "epoch": 0.7201152184349496,
      "grad_norm": 0.255859375,
      "learning_rate": 2.98051897890784e-07,
      "loss": 0.1637,
      "num_tokens": 322239993.0,
      "reward": 0.4036458507180214,
      "reward_std": 0.17170867137610912,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1875
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1541.4583435058594,
      "epoch": 0.7204992798847816,
      "grad_norm": 0.2080078125,
      "learning_rate": 2.975519692873638e-07,
      "loss": 0.0895,
      "num_tokens": 322402445.0,
      "reward": 0.5703125149011612,
      "reward_std": 0.2656290642917156,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1876
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.8854675292969,
      "epoch": 0.7208833413346135,
      "grad_norm": 0.2197265625,
      "learning_rate": 2.9705249493702927e-07,
      "loss": 0.0773,
      "num_tokens": 322586546.0,
      "reward": 0.3723958507180214,
      "reward_std": 0.11965805664658546,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791567325592,
      "step": 1877
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1435.8646240234375,
      "epoch": 0.7212674027844456,
      "grad_norm": 0.271484375,
      "learning_rate": 2.965534757385307e-07,
      "loss": 0.0936,
      "num_tokens": 322737981.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.17787999473512173,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 1878
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.0313110351562,
      "epoch": 0.7216514642342775,
      "grad_norm": 0.251953125,
      "learning_rate": 2.960549125897994e-07,
      "loss": 0.1228,
      "num_tokens": 322904192.0,
      "reward": 0.489583358168602,
      "reward_std": 0.2252888511866331,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687499925494194,
      "step": 1879
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1587.0104675292969,
      "epoch": 0.7220355256841094,
      "grad_norm": 0.1796875,
      "learning_rate": 2.955568063879464e-07,
      "loss": 0.0792,
      "num_tokens": 323071241.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.15812402544543147,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 1880
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1766.0208740234375,
      "epoch": 0.7224195871339414,
      "grad_norm": 0.1650390625,
      "learning_rate": 2.950591580292601e-07,
      "loss": 0.0497,
      "num_tokens": 323255451.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.18080234853550792,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1881
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1817.5104675292969,
      "epoch": 0.7228036485837734,
      "grad_norm": 0.2099609375,
      "learning_rate": 2.9456196840920517e-07,
      "loss": 0.1095,
      "num_tokens": 323445036.0,
      "reward": 0.393229179084301,
      "reward_std": 0.1684497408568859,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1882
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.9166870117188,
      "epoch": 0.7231877100336054,
      "grad_norm": 0.2080078125,
      "learning_rate": 2.940652384224207e-07,
      "loss": 0.1029,
      "num_tokens": 323624948.0,
      "reward": 0.598958358168602,
      "reward_std": 0.21517756395041943,
      "rewards/accuracy_reward": 0.1770833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1883
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.9375305175781,
      "epoch": 0.7235717714834373,
      "grad_norm": 0.185546875,
      "learning_rate": 2.9356896896271935e-07,
      "loss": 0.0829,
      "num_tokens": 323806014.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.1884565418586135,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 1884
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.2396240234375,
      "epoch": 0.7239558329332694,
      "grad_norm": 0.275390625,
      "learning_rate": 2.930731609230843e-07,
      "loss": 0.1043,
      "num_tokens": 323963405.0,
      "reward": 0.6119791865348816,
      "reward_std": 0.29823286831378937,
      "rewards/accuracy_reward": 0.14583333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 1885
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.0833740234375,
      "epoch": 0.7243398943831013,
      "grad_norm": 0.21484375,
      "learning_rate": 2.9257781519566885e-07,
      "loss": 0.1295,
      "num_tokens": 324141517.0,
      "reward": 0.440104179084301,
      "reward_std": 0.22526173666119576,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1886
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.6771240234375,
      "epoch": 0.7247239558329333,
      "grad_norm": 0.2001953125,
      "learning_rate": 2.9208293267179427e-07,
      "loss": 0.1101,
      "num_tokens": 324303366.0,
      "reward": 0.5755208432674408,
      "reward_std": 0.25486054457724094,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1887
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1681.5000610351562,
      "epoch": 0.7251080172827652,
      "grad_norm": 0.2099609375,
      "learning_rate": 2.9158851424194886e-07,
      "loss": 0.097,
      "num_tokens": 324480958.0,
      "reward": 0.513020858168602,
      "reward_std": 0.24564297124743462,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 1888
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.2604675292969,
      "epoch": 0.7254920787325972,
      "grad_norm": 0.2099609375,
      "learning_rate": 2.910945607957851e-07,
      "loss": 0.0861,
      "num_tokens": 324654135.0,
      "reward": 0.479166679084301,
      "reward_std": 0.19470908120274544,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1889
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1430.3125610351562,
      "epoch": 0.7258761401824292,
      "grad_norm": 0.2275390625,
      "learning_rate": 2.906010732221191e-07,
      "loss": 0.1198,
      "num_tokens": 324805309.0,
      "reward": 0.5468750298023224,
      "reward_std": 0.27575573697686195,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 1890
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1427.9062805175781,
      "epoch": 0.7262602016322611,
      "grad_norm": 0.2490234375,
      "learning_rate": 2.901080524089285e-07,
      "loss": 0.0948,
      "num_tokens": 324957748.0,
      "reward": 0.5703125223517418,
      "reward_std": 0.2730286717414856,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458283662796,
      "step": 1891
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1656.1667175292969,
      "epoch": 0.7266442630820932,
      "grad_norm": 0.1748046875,
      "learning_rate": 2.896154992433516e-07,
      "loss": 0.0837,
      "num_tokens": 325132596.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.13614366482943296,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 1892
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.2916870117188,
      "epoch": 0.7270283245319251,
      "grad_norm": 0.2451171875,
      "learning_rate": 2.8912341461168504e-07,
      "loss": 0.0548,
      "num_tokens": 325297536.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.20057901926338673,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1893
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1414.3646240234375,
      "epoch": 0.7274123859817571,
      "grad_norm": 0.25390625,
      "learning_rate": 2.886317993993815e-07,
      "loss": 0.0742,
      "num_tokens": 325448187.0,
      "reward": 0.6067708507180214,
      "reward_std": 0.2684948407113552,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5026041716337204,
      "step": 1894
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.5313110351562,
      "epoch": 0.727796447431589,
      "grad_norm": 0.21875,
      "learning_rate": 2.881406544910503e-07,
      "loss": 0.0847,
      "num_tokens": 325622742.0,
      "reward": 0.401041679084301,
      "reward_std": 0.13657010346651077,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1895
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.1562805175781,
      "epoch": 0.728180508881421,
      "grad_norm": 0.2138671875,
      "learning_rate": 2.876499807704537e-07,
      "loss": 0.1556,
      "num_tokens": 325788917.0,
      "reward": 0.5911458432674408,
      "reward_std": 0.3224554881453514,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 1896
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.2916870117188,
      "epoch": 0.728564570331253,
      "grad_norm": 0.208984375,
      "learning_rate": 2.871597791205062e-07,
      "loss": 0.0865,
      "num_tokens": 325954969.0,
      "reward": 0.447916679084301,
      "reward_std": 0.16243189573287964,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 1897
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.3125305175781,
      "epoch": 0.7289486317810849,
      "grad_norm": 0.2294921875,
      "learning_rate": 2.866700504232729e-07,
      "loss": 0.0695,
      "num_tokens": 326117687.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.22976182401180267,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1898
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1599.5521240234375,
      "epoch": 0.729332693230917,
      "grad_norm": 0.2158203125,
      "learning_rate": 2.861807955599682e-07,
      "loss": 0.0975,
      "num_tokens": 326283612.0,
      "reward": 0.5286458432674408,
      "reward_std": 0.2599259652197361,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1899
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1438.0938110351562,
      "epoch": 0.7297167546807489,
      "grad_norm": 0.2421875,
      "learning_rate": 2.856920154109535e-07,
      "loss": 0.1899,
      "num_tokens": 326434005.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.2979373373091221,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687500074505806,
      "step": 1900
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1735.5417175292969,
      "epoch": 0.7301008161305809,
      "grad_norm": 0.193359375,
      "learning_rate": 2.8520371085573616e-07,
      "loss": 0.1214,
      "num_tokens": 326615521.0,
      "reward": 0.4348958358168602,
      "reward_std": 0.2034756913781166,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 1901
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.6770935058594,
      "epoch": 0.7304848775804129,
      "grad_norm": 0.216796875,
      "learning_rate": 2.8471588277296734e-07,
      "loss": 0.078,
      "num_tokens": 326781178.0,
      "reward": 0.4843750074505806,
      "reward_std": 0.21608075499534607,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 1902
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1485.5104675292969,
      "epoch": 0.7308689390302449,
      "grad_norm": 0.236328125,
      "learning_rate": 2.8422853204044177e-07,
      "loss": 0.1317,
      "num_tokens": 326937603.0,
      "reward": 0.479166679084301,
      "reward_std": 0.17781700380146503,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687500074505806,
      "step": 1903
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.8958740234375,
      "epoch": 0.7312530004800768,
      "grad_norm": 0.2353515625,
      "learning_rate": 2.8374165953509444e-07,
      "loss": 0.1119,
      "num_tokens": 327100977.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.2578662149608135,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1904
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.2500305175781,
      "epoch": 0.7316370619299087,
      "grad_norm": 0.251953125,
      "learning_rate": 2.832552661330001e-07,
      "loss": 0.1353,
      "num_tokens": 327274305.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.123107785359025,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 1905
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1906.3437805175781,
      "epoch": 0.7320211233797408,
      "grad_norm": 0.150390625,
      "learning_rate": 2.827693527093712e-07,
      "loss": 0.0342,
      "num_tokens": 327472050.0,
      "reward": 0.416666679084301,
      "reward_std": 0.16187759023159742,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333283662796,
      "step": 1906
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1293.7500305175781,
      "epoch": 0.7324051848295727,
      "grad_norm": 0.2890625,
      "learning_rate": 2.8228392013855684e-07,
      "loss": 0.1204,
      "num_tokens": 327611178.0,
      "reward": 0.645833358168602,
      "reward_std": 0.3322886936366558,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5104166641831398,
      "step": 1907
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1481.2292175292969,
      "epoch": 0.7327892462794047,
      "grad_norm": 0.259765625,
      "learning_rate": 2.81798969294041e-07,
      "loss": 0.1229,
      "num_tokens": 327769016.0,
      "reward": 0.5468750223517418,
      "reward_std": 0.2583271749317646,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250074505806,
      "step": 1908
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.5521240234375,
      "epoch": 0.7331733077292367,
      "grad_norm": 0.26953125,
      "learning_rate": 2.813145010484401e-07,
      "loss": 0.0887,
      "num_tokens": 327943949.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.18151417654007673,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 1909
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1524.2188110351562,
      "epoch": 0.7335573691790687,
      "grad_norm": 0.220703125,
      "learning_rate": 2.8083051627350294e-07,
      "loss": 0.1134,
      "num_tokens": 328105994.0,
      "reward": 0.486979179084301,
      "reward_std": 0.17941556870937347,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 1910
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1233.9896240234375,
      "epoch": 0.7339414306289006,
      "grad_norm": 0.298828125,
      "learning_rate": 2.80347015840108e-07,
      "loss": 0.033,
      "num_tokens": 328237649.0,
      "reward": 0.7864583432674408,
      "reward_std": 0.3455715477466583,
      "rewards/accuracy_reward": 0.2708333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.515625,
      "step": 1911
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1392.0938110351562,
      "epoch": 0.7343254920787325,
      "grad_norm": 0.291015625,
      "learning_rate": 2.7986400061826255e-07,
      "loss": 0.1189,
      "num_tokens": 328388242.0,
      "reward": 0.6093750149011612,
      "reward_std": 0.33135484904050827,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5052083358168602,
      "step": 1912
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.8229675292969,
      "epoch": 0.7347095535285646,
      "grad_norm": 0.2265625,
      "learning_rate": 2.7938147147710004e-07,
      "loss": 0.0749,
      "num_tokens": 328558713.0,
      "reward": 0.591145858168602,
      "reward_std": 0.24435248225927353,
      "rewards/accuracy_reward": 0.21875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 1913
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1512.9688110351562,
      "epoch": 0.7350936149783965,
      "grad_norm": 0.21875,
      "learning_rate": 2.788994292848802e-07,
      "loss": 0.1436,
      "num_tokens": 328717790.0,
      "reward": 0.6067708432674408,
      "reward_std": 0.23635784164071083,
      "rewards/accuracy_reward": 0.1562499962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208432674408,
      "step": 1914
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.7292175292969,
      "epoch": 0.7354776764282285,
      "grad_norm": 0.2001953125,
      "learning_rate": 2.784178749089859e-07,
      "loss": 0.0888,
      "num_tokens": 328897820.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.1423097588121891,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1915
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1759.3646240234375,
      "epoch": 0.7358617378780605,
      "grad_norm": 0.1796875,
      "learning_rate": 2.7793680921592276e-07,
      "loss": 0.1074,
      "num_tokens": 329082319.0,
      "reward": 0.510416679084301,
      "reward_std": 0.19225679337978363,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 1916
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1648.1666870117188,
      "epoch": 0.7362457993278925,
      "grad_norm": 0.2119140625,
      "learning_rate": 2.7745623307131625e-07,
      "loss": 0.0827,
      "num_tokens": 329258255.0,
      "reward": 0.4192708358168602,
      "reward_std": 0.17826450243592262,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1917
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1433.1354675292969,
      "epoch": 0.7366298607777244,
      "grad_norm": 0.25,
      "learning_rate": 2.769761473399119e-07,
      "loss": 0.0978,
      "num_tokens": 329414612.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.19903316348791122,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687500074505806,
      "step": 1918
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1645.0729675292969,
      "epoch": 0.7370139222275565,
      "grad_norm": 0.25390625,
      "learning_rate": 2.7649655288557237e-07,
      "loss": 0.1195,
      "num_tokens": 329585899.0,
      "reward": 0.4921875074505806,
      "reward_std": 0.18883217498660088,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1919
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1864.4167175292969,
      "epoch": 0.7373979836773884,
      "grad_norm": 0.1884765625,
      "learning_rate": 2.76017450571276e-07,
      "loss": 0.0721,
      "num_tokens": 329782883.0,
      "reward": 0.3723958358168602,
      "reward_std": 0.16309788078069687,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 1920
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1399.5521240234375,
      "epoch": 0.7377820451272203,
      "grad_norm": 0.240234375,
      "learning_rate": 2.75538841259116e-07,
      "loss": 0.0941,
      "num_tokens": 329930104.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.123567258939147,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 1921
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.5833740234375,
      "epoch": 0.7381661065770523,
      "grad_norm": 0.271484375,
      "learning_rate": 2.750607258102983e-07,
      "loss": 0.1278,
      "num_tokens": 330088576.0,
      "reward": 0.4947916716337204,
      "reward_std": 0.19046092871576548,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1922
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.1458740234375,
      "epoch": 0.7385501680268843,
      "grad_norm": 0.1640625,
      "learning_rate": 2.7458310508514084e-07,
      "loss": 0.1007,
      "num_tokens": 330268782.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.18259135354310274,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 1923
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1765.9687805175781,
      "epoch": 0.7389342294767163,
      "grad_norm": 0.1884765625,
      "learning_rate": 2.7410597994306974e-07,
      "loss": 0.0843,
      "num_tokens": 330454179.0,
      "reward": 0.3906250074505806,
      "reward_std": 0.13927297852933407,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906250074505806,
      "step": 1924
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1643.1666870117188,
      "epoch": 0.7393182909265482,
      "grad_norm": 0.2265625,
      "learning_rate": 2.7362935124262123e-07,
      "loss": 0.0975,
      "num_tokens": 330627763.0,
      "reward": 0.4843750223517418,
      "reward_std": 0.2667698413133621,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 1925
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.9271240234375,
      "epoch": 0.7397023523763803,
      "grad_norm": 0.244140625,
      "learning_rate": 2.731532198414368e-07,
      "loss": 0.0867,
      "num_tokens": 330797428.0,
      "reward": 0.552083358168602,
      "reward_std": 0.3102064412087202,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 1926
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.3021240234375,
      "epoch": 0.7400864138262122,
      "grad_norm": 0.244140625,
      "learning_rate": 2.726775865962644e-07,
      "loss": 0.096,
      "num_tokens": 330966097.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.1505938656628132,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 1927
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1673.6458740234375,
      "epoch": 0.7404704752760441,
      "grad_norm": 0.2373046875,
      "learning_rate": 2.7220245236295427e-07,
      "loss": 0.1456,
      "num_tokens": 331141735.0,
      "reward": 0.5729166939854622,
      "reward_std": 0.2962733507156372,
      "rewards/accuracy_reward": 0.18750000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 1928
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.1354370117188,
      "epoch": 0.7408545367258761,
      "grad_norm": 0.1826171875,
      "learning_rate": 2.717278179964596e-07,
      "loss": 0.0844,
      "num_tokens": 331316580.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.22974305506795645,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1929
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.385498046875,
      "epoch": 0.7412385981757081,
      "grad_norm": 0.177734375,
      "learning_rate": 2.7125368435083387e-07,
      "loss": 0.0822,
      "num_tokens": 331495801.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.12814229633659124,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1930
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.2292175292969,
      "epoch": 0.7416226596255401,
      "grad_norm": 0.25,
      "learning_rate": 2.7078005227922983e-07,
      "loss": 0.1054,
      "num_tokens": 331661679.0,
      "reward": 0.5286458432674408,
      "reward_std": 0.18251501582562923,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 1931
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.9687805175781,
      "epoch": 0.742006721075372,
      "grad_norm": 0.2490234375,
      "learning_rate": 2.7030692263389706e-07,
      "loss": 0.1263,
      "num_tokens": 331815148.0,
      "reward": 0.6510416716337204,
      "reward_std": 0.33156759664416313,
      "rewards/accuracy_reward": 0.2083333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 1932
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1631.3958740234375,
      "epoch": 0.7423907825252041,
      "grad_norm": 0.228515625,
      "learning_rate": 2.698342962661818e-07,
      "loss": 0.1375,
      "num_tokens": 331984434.0,
      "reward": 0.5546875223517418,
      "reward_std": 0.29646942019462585,
      "rewards/accuracy_reward": 0.12500000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875074505806,
      "step": 1933
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.4583740234375,
      "epoch": 0.742774843975036,
      "grad_norm": 0.2099609375,
      "learning_rate": 2.6936217402652407e-07,
      "loss": 0.1084,
      "num_tokens": 332164902.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.25478778034448624,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 1934
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1789.3646240234375,
      "epoch": 0.743158905424868,
      "grad_norm": 0.1572265625,
      "learning_rate": 2.688905567644576e-07,
      "loss": 0.0587,
      "num_tokens": 332350561.0,
      "reward": 0.3854166865348816,
      "reward_std": 0.14598759450018406,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 1935
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.8854675292969,
      "epoch": 0.7435429668747,
      "grad_norm": 0.185546875,
      "learning_rate": 2.684194453286063e-07,
      "loss": 0.1049,
      "num_tokens": 332517110.0,
      "reward": 0.5546875149011612,
      "reward_std": 0.24276924040168524,
      "rewards/accuracy_reward": 0.12500000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 1936
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.7083740234375,
      "epoch": 0.743927028324532,
      "grad_norm": 0.2236328125,
      "learning_rate": 2.679488405666848e-07,
      "loss": 0.1092,
      "num_tokens": 332698514.0,
      "reward": 0.4427083358168602,
      "reward_std": 0.20237311720848083,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 1937
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1378.5729217529297,
      "epoch": 0.7443110897743639,
      "grad_norm": 0.2314453125,
      "learning_rate": 2.6747874332549613e-07,
      "loss": 0.0696,
      "num_tokens": 332844089.0,
      "reward": 0.6666666865348816,
      "reward_std": 0.32813334465026855,
      "rewards/accuracy_reward": 0.20833333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333283662796,
      "step": 1938
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.5937805175781,
      "epoch": 0.7446951512241958,
      "grad_norm": 0.1875,
      "learning_rate": 2.6700915445092917e-07,
      "loss": 0.1432,
      "num_tokens": 333029618.0,
      "reward": 0.354166679084301,
      "reward_std": 0.1285129552707076,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3437499925494194,
      "step": 1939
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.0521240234375,
      "epoch": 0.7450792126740279,
      "grad_norm": 0.1923828125,
      "learning_rate": 2.665400747879589e-07,
      "loss": 0.1245,
      "num_tokens": 333200327.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.256135243922472,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541716337204,
      "step": 1940
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.6771240234375,
      "epoch": 0.7454632741238598,
      "grad_norm": 0.21875,
      "learning_rate": 2.660715051806436e-07,
      "loss": 0.0787,
      "num_tokens": 333366408.0,
      "reward": 0.4635416716337204,
      "reward_std": 0.1938371304422617,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1941
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.8229522705078,
      "epoch": 0.7458473355736918,
      "grad_norm": 0.236328125,
      "learning_rate": 2.656034464721242e-07,
      "loss": 0.0544,
      "num_tokens": 333526799.0,
      "reward": 0.4869791716337204,
      "reward_std": 0.19496204983443022,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1942
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1330.4375610351562,
      "epoch": 0.7462313970235238,
      "grad_norm": 0.265625,
      "learning_rate": 2.6513589950462147e-07,
      "loss": 0.0643,
      "num_tokens": 333667777.0,
      "reward": 0.6744791939854622,
      "reward_std": 0.28633948788046837,
      "rewards/accuracy_reward": 0.1875000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4869791641831398,
      "step": 1943
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.6354370117188,
      "epoch": 0.7466154584733558,
      "grad_norm": 0.20703125,
      "learning_rate": 2.646688651194364e-07,
      "loss": 0.094,
      "num_tokens": 333843406.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.28517812117934227,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 1944
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.1250305175781,
      "epoch": 0.7469995199231877,
      "grad_norm": 0.197265625,
      "learning_rate": 2.642023441569468e-07,
      "loss": 0.0541,
      "num_tokens": 334018170.0,
      "reward": 0.408854179084301,
      "reward_std": 0.15721799992024899,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041567325592,
      "step": 1945
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1554.9271240234375,
      "epoch": 0.7473835813730196,
      "grad_norm": 0.251953125,
      "learning_rate": 2.637363374566074e-07,
      "loss": 0.0636,
      "num_tokens": 334181467.0,
      "reward": 0.5078125074505806,
      "reward_std": 0.2503918185830116,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1946
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1960.6042175292969,
      "epoch": 0.7477676428228517,
      "grad_norm": 0.1787109375,
      "learning_rate": 2.6327084585694667e-07,
      "loss": 0.0529,
      "num_tokens": 334387085.0,
      "reward": 0.3281250149011612,
      "reward_std": 0.12625435180962086,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3072916641831398,
      "step": 1947
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.5625305175781,
      "epoch": 0.7481517042726836,
      "grad_norm": 0.2021484375,
      "learning_rate": 2.62805870195567e-07,
      "loss": 0.1331,
      "num_tokens": 334553987.0,
      "reward": 0.5989583432674408,
      "reward_std": 0.2831559255719185,
      "rewards/accuracy_reward": 0.1562500037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 1948
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1670.0833740234375,
      "epoch": 0.7485357657225156,
      "grad_norm": 0.232421875,
      "learning_rate": 2.6234141130914187e-07,
      "loss": 0.1219,
      "num_tokens": 334729187.0,
      "reward": 0.510416679084301,
      "reward_std": 0.259736031293869,
      "rewards/accuracy_reward": 0.10416666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 1949
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1760.9063110351562,
      "epoch": 0.7489198271723476,
      "grad_norm": 0.2197265625,
      "learning_rate": 2.6187747003341555e-07,
      "loss": 0.1581,
      "num_tokens": 334914426.0,
      "reward": 0.3828125074505806,
      "reward_std": 0.16016202233731747,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 1950
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.2604370117188,
      "epoch": 0.7493038886221796,
      "grad_norm": 0.177734375,
      "learning_rate": 2.6141404720319984e-07,
      "loss": 0.0816,
      "num_tokens": 335100963.0,
      "reward": 0.361979179084301,
      "reward_std": 0.09820624394342303,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1951
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.4375305175781,
      "epoch": 0.7496879500720115,
      "grad_norm": 0.2431640625,
      "learning_rate": 2.609511436523746e-07,
      "loss": 0.1563,
      "num_tokens": 335269661.0,
      "reward": 0.4505208358168602,
      "reward_std": 0.20795863308012486,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 1952
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1458.0520935058594,
      "epoch": 0.7500720115218434,
      "grad_norm": 0.23828125,
      "learning_rate": 2.604887602138851e-07,
      "loss": 0.1932,
      "num_tokens": 335421906.0,
      "reward": 0.5364583507180214,
      "reward_std": 0.3020751914009452,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583432674408,
      "step": 1953
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1508.7500305175781,
      "epoch": 0.7504560729716755,
      "grad_norm": 0.2353515625,
      "learning_rate": 2.600268977197405e-07,
      "loss": 0.1452,
      "num_tokens": 335584410.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.23488867841660976,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833432674408,
      "step": 1954
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.0729370117188,
      "epoch": 0.7508401344215074,
      "grad_norm": 0.19140625,
      "learning_rate": 2.5956555700101286e-07,
      "loss": 0.1114,
      "num_tokens": 335764857.0,
      "reward": 0.4687500074505806,
      "reward_std": 0.15895803458988667,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1955
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1495.6250610351562,
      "epoch": 0.7512241958713394,
      "grad_norm": 0.2353515625,
      "learning_rate": 2.591047388878348e-07,
      "loss": 0.1413,
      "num_tokens": 335924853.0,
      "reward": 0.5677083507180214,
      "reward_std": 0.30615071952342987,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 1956
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.6250305175781,
      "epoch": 0.7516082573211714,
      "grad_norm": 0.220703125,
      "learning_rate": 2.5864444420939943e-07,
      "loss": 0.0907,
      "num_tokens": 336083865.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.17658151499927044,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 1957
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1788.2812805175781,
      "epoch": 0.7519923187710034,
      "grad_norm": 0.1953125,
      "learning_rate": 2.5818467379395724e-07,
      "loss": 0.1464,
      "num_tokens": 336268756.0,
      "reward": 0.432291679084301,
      "reward_std": 0.2540348246693611,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 1958
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.2812805175781,
      "epoch": 0.7523763802208353,
      "grad_norm": 0.2109375,
      "learning_rate": 2.577254284688158e-07,
      "loss": 0.09,
      "num_tokens": 336457215.0,
      "reward": 0.385416679084301,
      "reward_std": 0.16092536970973015,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1959
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.6146545410156,
      "epoch": 0.7527604416706674,
      "grad_norm": 0.203125,
      "learning_rate": 2.5726670906033744e-07,
      "loss": 0.0824,
      "num_tokens": 336622242.0,
      "reward": 0.559895858168602,
      "reward_std": 0.2489471174776554,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 1960
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1769.7083740234375,
      "epoch": 0.7531445031204993,
      "grad_norm": 0.19921875,
      "learning_rate": 2.568085163939389e-07,
      "loss": 0.1295,
      "num_tokens": 336804726.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.22833962365984917,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1961
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1388.0521240234375,
      "epoch": 0.7535285645703312,
      "grad_norm": 0.29296875,
      "learning_rate": 2.5635085129408794e-07,
      "loss": 0.079,
      "num_tokens": 336951899.0,
      "reward": 0.588541679084301,
      "reward_std": 0.26037221401929855,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583283662796,
      "step": 1962
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.7187805175781,
      "epoch": 0.7539126260201632,
      "grad_norm": 0.1865234375,
      "learning_rate": 2.558937145843042e-07,
      "loss": 0.077,
      "num_tokens": 337128160.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.18111669644713402,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 1963
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.2500305175781,
      "epoch": 0.7542966874699952,
      "grad_norm": 0.2138671875,
      "learning_rate": 2.5543710708715583e-07,
      "loss": 0.1438,
      "num_tokens": 337297416.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.1835273839533329,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 1964
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.84375,
      "epoch": 0.7546807489198272,
      "grad_norm": 0.24609375,
      "learning_rate": 2.549810296242592e-07,
      "loss": 0.1274,
      "num_tokens": 337467233.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.18565725162625313,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 1965
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.4687805175781,
      "epoch": 0.7550648103696591,
      "grad_norm": 0.216796875,
      "learning_rate": 2.545254830162762e-07,
      "loss": 0.1541,
      "num_tokens": 337632990.0,
      "reward": 0.5416666865348816,
      "reward_std": 0.31388354301452637,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 1966
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.6250305175781,
      "epoch": 0.7554488718194912,
      "grad_norm": 0.1923828125,
      "learning_rate": 2.540704680829141e-07,
      "loss": 0.1088,
      "num_tokens": 337809618.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.16935035958886147,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 1967
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.4479370117188,
      "epoch": 0.7558329332693231,
      "grad_norm": 0.1953125,
      "learning_rate": 2.5361598564292375e-07,
      "loss": 0.0819,
      "num_tokens": 337993125.0,
      "reward": 0.401041679084301,
      "reward_std": 0.15123221091926098,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 1968
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.3021545410156,
      "epoch": 0.756216994719155,
      "grad_norm": 0.2138671875,
      "learning_rate": 2.5316203651409706e-07,
      "loss": 0.1544,
      "num_tokens": 338169786.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.2006763406097889,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 1969
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1639.3646240234375,
      "epoch": 0.756601056168987,
      "grad_norm": 0.2236328125,
      "learning_rate": 2.527086215132667e-07,
      "loss": 0.1123,
      "num_tokens": 338341157.0,
      "reward": 0.447916679084301,
      "reward_std": 0.19686272367835045,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1970
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.7500610351562,
      "epoch": 0.756985117618819,
      "grad_norm": 0.1806640625,
      "learning_rate": 2.5225574145630424e-07,
      "loss": 0.1105,
      "num_tokens": 338529725.0,
      "reward": 0.4140625149011612,
      "reward_std": 0.1826114933937788,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 1971
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1455.5729675292969,
      "epoch": 0.757369179068651,
      "grad_norm": 0.2734375,
      "learning_rate": 2.518033971581187e-07,
      "loss": 0.073,
      "num_tokens": 338684460.0,
      "reward": 0.559895858168602,
      "reward_std": 0.29532628506422043,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765624925494194,
      "step": 1972
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.7708740234375,
      "epoch": 0.7577532405184829,
      "grad_norm": 0.2353515625,
      "learning_rate": 2.51351589432655e-07,
      "loss": 0.1397,
      "num_tokens": 338844806.0,
      "reward": 0.541666679084301,
      "reward_std": 0.3030930161476135,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 1973
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1678.4792175292969,
      "epoch": 0.758137301968315,
      "grad_norm": 0.2119140625,
      "learning_rate": 2.5090031909289256e-07,
      "loss": 0.082,
      "num_tokens": 339020596.0,
      "reward": 0.3880208358168602,
      "reward_std": 0.09812546242028475,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 1974
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.072998046875,
      "epoch": 0.7585213634181469,
      "grad_norm": 0.189453125,
      "learning_rate": 2.504495869508436e-07,
      "loss": 0.1615,
      "num_tokens": 339206195.0,
      "reward": 0.393229179084301,
      "reward_std": 0.1886129342019558,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 1975
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.6666870117188,
      "epoch": 0.7589054248679789,
      "grad_norm": 0.1953125,
      "learning_rate": 2.4999939381755245e-07,
      "loss": 0.0989,
      "num_tokens": 339381483.0,
      "reward": 0.5078125223517418,
      "reward_std": 0.2655684407800436,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 1976
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.9271240234375,
      "epoch": 0.7592894863178109,
      "grad_norm": 0.2431640625,
      "learning_rate": 2.495497405030932e-07,
      "loss": 0.0819,
      "num_tokens": 339542108.0,
      "reward": 0.4401041716337204,
      "reward_std": 0.1266075475141406,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 1977
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.5000610351562,
      "epoch": 0.7596735477676428,
      "grad_norm": 0.1943359375,
      "learning_rate": 2.4910062781656853e-07,
      "loss": 0.1271,
      "num_tokens": 339728156.0,
      "reward": 0.447916679084301,
      "reward_std": 0.22766397148370743,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1978
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1636.3437805175781,
      "epoch": 0.7600576092174748,
      "grad_norm": 0.236328125,
      "learning_rate": 2.4865205656610837e-07,
      "loss": 0.0593,
      "num_tokens": 339901285.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.2460012286901474,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 1979
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.8021240234375,
      "epoch": 0.7604416706673067,
      "grad_norm": 0.224609375,
      "learning_rate": 2.4820402755886875e-07,
      "loss": 0.1026,
      "num_tokens": 340084642.0,
      "reward": 0.5416666865348816,
      "reward_std": 0.3035639710724354,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3749999925494194,
      "step": 1980
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.7292175292969,
      "epoch": 0.7608257321171388,
      "grad_norm": 0.2060546875,
      "learning_rate": 2.477565416010293e-07,
      "loss": 0.1019,
      "num_tokens": 340264256.0,
      "reward": 0.4140625149011612,
      "reward_std": 0.16728424932807684,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 1981
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.9584045410156,
      "epoch": 0.7612097935669707,
      "grad_norm": 0.23046875,
      "learning_rate": 2.4730959949779304e-07,
      "loss": 0.169,
      "num_tokens": 340433284.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.16701499745249748,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 1982
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1516.8646240234375,
      "epoch": 0.7615938550168027,
      "grad_norm": 0.228515625,
      "learning_rate": 2.4686320205338435e-07,
      "loss": 0.0914,
      "num_tokens": 340594279.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.2563681975007057,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1983
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.9687805175781,
      "epoch": 0.7619779164666347,
      "grad_norm": 0.220703125,
      "learning_rate": 2.464173500710474e-07,
      "loss": 0.0635,
      "num_tokens": 340757508.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.186994269490242,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 1984
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.7708740234375,
      "epoch": 0.7623619779164666,
      "grad_norm": 0.19140625,
      "learning_rate": 2.4597204435304484e-07,
      "loss": 0.1199,
      "num_tokens": 340933382.0,
      "reward": 0.494791679084301,
      "reward_std": 0.2624977342784405,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 1985
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1816.4479370117188,
      "epoch": 0.7627460393662986,
      "grad_norm": 0.1875,
      "learning_rate": 2.4552728570065623e-07,
      "loss": 0.0589,
      "num_tokens": 341122305.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.2618717420846224,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 1986
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1707.8854675292969,
      "epoch": 0.7631301008161305,
      "grad_norm": 0.1982421875,
      "learning_rate": 2.450830749141773e-07,
      "loss": 0.1016,
      "num_tokens": 341302134.0,
      "reward": 0.453125,
      "reward_std": 0.20834481995552778,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 1987
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.7083740234375,
      "epoch": 0.7635141622659626,
      "grad_norm": 0.255859375,
      "learning_rate": 2.446394127929174e-07,
      "loss": 0.1228,
      "num_tokens": 341463330.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.17213767766952515,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 1988
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1390.9167175292969,
      "epoch": 0.7638982237157945,
      "grad_norm": 0.267578125,
      "learning_rate": 2.441963001351988e-07,
      "loss": 0.0991,
      "num_tokens": 341609778.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.1818670593202114,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 1989
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.7187805175781,
      "epoch": 0.7642822851656265,
      "grad_norm": 0.283203125,
      "learning_rate": 2.43753737738355e-07,
      "loss": 0.1146,
      "num_tokens": 341765999.0,
      "reward": 0.583333358168602,
      "reward_std": 0.3325486332178116,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333432674408,
      "step": 1990
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.9479675292969,
      "epoch": 0.7646663466154585,
      "grad_norm": 0.236328125,
      "learning_rate": 2.4331172639872974e-07,
      "loss": 0.1246,
      "num_tokens": 341931258.0,
      "reward": 0.5494791716337204,
      "reward_std": 0.25273212045431137,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458283662796,
      "step": 1991
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.1770935058594,
      "epoch": 0.7650504080652905,
      "grad_norm": 0.25,
      "learning_rate": 2.4287026691167465e-07,
      "loss": 0.1087,
      "num_tokens": 342098363.0,
      "reward": 0.5937500298023224,
      "reward_std": 0.3710508942604065,
      "rewards/accuracy_reward": 0.17708333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 1992
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.8229370117188,
      "epoch": 0.7654344695151224,
      "grad_norm": 0.173828125,
      "learning_rate": 2.4242936007154865e-07,
      "loss": 0.0936,
      "num_tokens": 342261986.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.15856588911265135,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1993
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1207.6562957763672,
      "epoch": 0.7658185309649544,
      "grad_norm": 0.345703125,
      "learning_rate": 2.4198900667171614e-07,
      "loss": 0.126,
      "num_tokens": 342392681.0,
      "reward": 0.6380208432674408,
      "reward_std": 0.23681407421827316,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4817708358168602,
      "step": 1994
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1718.0625610351562,
      "epoch": 0.7662025924147864,
      "grad_norm": 0.2431640625,
      "learning_rate": 2.4154920750454584e-07,
      "loss": 0.0809,
      "num_tokens": 342572039.0,
      "reward": 0.5625000074505806,
      "reward_std": 0.30172617733478546,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 1995
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.6459045410156,
      "epoch": 0.7665866538646183,
      "grad_norm": 0.2119140625,
      "learning_rate": 2.41109963361409e-07,
      "loss": 0.1398,
      "num_tokens": 342740925.0,
      "reward": 0.5572916716337204,
      "reward_std": 0.27049797400832176,
      "rewards/accuracy_reward": 0.15625000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 1996
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.5416870117188,
      "epoch": 0.7669707153144503,
      "grad_norm": 0.21875,
      "learning_rate": 2.406712750326781e-07,
      "loss": 0.1092,
      "num_tokens": 342915489.0,
      "reward": 0.4921875074505806,
      "reward_std": 0.21090780198574066,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 1997
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.3750305175781,
      "epoch": 0.7673547767642823,
      "grad_norm": 0.265625,
      "learning_rate": 2.4023314330772595e-07,
      "loss": 0.1366,
      "num_tokens": 343078541.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.21366377361118793,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041567325592,
      "step": 1998
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1524.3333435058594,
      "epoch": 0.7677388382141143,
      "grad_norm": 0.236328125,
      "learning_rate": 2.3979556897492315e-07,
      "loss": 0.05,
      "num_tokens": 343242621.0,
      "reward": 0.4687500074505806,
      "reward_std": 0.1548553928732872,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 1999
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1546.1979675292969,
      "epoch": 0.7681228996639462,
      "grad_norm": 0.2216796875,
      "learning_rate": 2.393585528216378e-07,
      "loss": 0.1133,
      "num_tokens": 343404952.0,
      "reward": 0.6250000223517418,
      "reward_std": 0.3791586570441723,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4791666716337204,
      "step": 2000
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.3021240234375,
      "epoch": 0.7685069611137783,
      "grad_norm": 0.25,
      "learning_rate": 2.3892209563423334e-07,
      "loss": 0.1076,
      "num_tokens": 343574581.0,
      "reward": 0.4192708358168602,
      "reward_std": 0.19528841972351074,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2001
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.5938110351562,
      "epoch": 0.7688910225636102,
      "grad_norm": 0.25390625,
      "learning_rate": 2.384861981980678e-07,
      "loss": 0.1133,
      "num_tokens": 343727486.0,
      "reward": 0.583333358168602,
      "reward_std": 0.31050725281238556,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4895833358168602,
      "step": 2002
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1854.5208435058594,
      "epoch": 0.7692750840134421,
      "grad_norm": 0.1650390625,
      "learning_rate": 2.3805086129749154e-07,
      "loss": 0.104,
      "num_tokens": 343919272.0,
      "reward": 0.393229179084301,
      "reward_std": 0.12454832065850496,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3307291641831398,
      "step": 2003
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1699.9271240234375,
      "epoch": 0.7696591454632741,
      "grad_norm": 0.2392578125,
      "learning_rate": 2.376160857158466e-07,
      "loss": 0.1644,
      "num_tokens": 344095937.0,
      "reward": 0.3984375149011612,
      "reward_std": 0.16653206013143063,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2004
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.2917175292969,
      "epoch": 0.7700432069131061,
      "grad_norm": 0.244140625,
      "learning_rate": 2.3718187223546473e-07,
      "loss": 0.0486,
      "num_tokens": 344254957.0,
      "reward": 0.4817708432674408,
      "reward_std": 0.1984202116727829,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 2005
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.8854370117188,
      "epoch": 0.7704272683629381,
      "grad_norm": 0.1728515625,
      "learning_rate": 2.367482216376666e-07,
      "loss": 0.0887,
      "num_tokens": 344442362.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.16649020183831453,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 2006
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.5416870117188,
      "epoch": 0.77081132981277,
      "grad_norm": 0.236328125,
      "learning_rate": 2.363151347027598e-07,
      "loss": 0.0415,
      "num_tokens": 344594046.0,
      "reward": 0.6250000149011612,
      "reward_std": 0.2637949362397194,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2007
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.2708740234375,
      "epoch": 0.7711953912626021,
      "grad_norm": 0.1611328125,
      "learning_rate": 2.358826122100376e-07,
      "loss": 0.0847,
      "num_tokens": 344776528.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.204360481351614,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 2008
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1424.0521240234375,
      "epoch": 0.771579452712434,
      "grad_norm": 0.2265625,
      "learning_rate": 2.3545065493777762e-07,
      "loss": 0.1176,
      "num_tokens": 344928069.0,
      "reward": 0.6015625149011612,
      "reward_std": 0.3147554323077202,
      "rewards/accuracy_reward": 0.10416666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4973958283662796,
      "step": 2009
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1494.2917175292969,
      "epoch": 0.7719635141622659,
      "grad_norm": 0.240234375,
      "learning_rate": 2.3501926366324085e-07,
      "loss": 0.1338,
      "num_tokens": 345086449.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.26935023814439774,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 2010
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1697.6354675292969,
      "epoch": 0.7723475756120979,
      "grad_norm": 0.263671875,
      "learning_rate": 2.345884391626693e-07,
      "loss": 0.1344,
      "num_tokens": 345263750.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.18320501036942005,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 2011
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.6354370117188,
      "epoch": 0.7727316370619299,
      "grad_norm": 0.197265625,
      "learning_rate": 2.3415818221128517e-07,
      "loss": 0.148,
      "num_tokens": 345440459.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.22941840440034866,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2012
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1618.9479675292969,
      "epoch": 0.7731156985117619,
      "grad_norm": 0.22265625,
      "learning_rate": 2.3372849358328992e-07,
      "loss": 0.135,
      "num_tokens": 345608326.0,
      "reward": 0.4401041865348816,
      "reward_std": 0.187759917229414,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 2013
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1825.5833740234375,
      "epoch": 0.7734997599615938,
      "grad_norm": 0.1796875,
      "learning_rate": 2.332993740518619e-07,
      "loss": 0.1141,
      "num_tokens": 345799654.0,
      "reward": 0.3567708432674408,
      "reward_std": 0.11323343170806766,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541641831398,
      "step": 2014
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1780.1250305175781,
      "epoch": 0.7738838214114259,
      "grad_norm": 0.2080078125,
      "learning_rate": 2.3287082438915551e-07,
      "loss": 0.098,
      "num_tokens": 345986146.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.1526774987578392,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3593750074505806,
      "step": 2015
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1763.2187805175781,
      "epoch": 0.7742678828612578,
      "grad_norm": 0.1787109375,
      "learning_rate": 2.3244284536629967e-07,
      "loss": 0.0659,
      "num_tokens": 346169615.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.11963889934122562,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 2016
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1671.9375610351562,
      "epoch": 0.7746519443110897,
      "grad_norm": 0.21875,
      "learning_rate": 2.3201543775339682e-07,
      "loss": 0.1106,
      "num_tokens": 346343497.0,
      "reward": 0.455729179084301,
      "reward_std": 0.19984444975852966,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 2017
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1472.2187805175781,
      "epoch": 0.7750360057609218,
      "grad_norm": 0.234375,
      "learning_rate": 2.31588602319521e-07,
      "loss": 0.1352,
      "num_tokens": 346498798.0,
      "reward": 0.5651041716337204,
      "reward_std": 0.21875469014048576,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 2018
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.3333435058594,
      "epoch": 0.7754200672107537,
      "grad_norm": 0.2177734375,
      "learning_rate": 2.3116233983271664e-07,
      "loss": 0.1209,
      "num_tokens": 346673046.0,
      "reward": 0.424479179084301,
      "reward_std": 0.14281836478039622,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 2019
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1656.9479675292969,
      "epoch": 0.7758041286605857,
      "grad_norm": 0.2236328125,
      "learning_rate": 2.3073665105999705e-07,
      "loss": 0.1267,
      "num_tokens": 346844937.0,
      "reward": 0.4739583507180214,
      "reward_std": 0.21496954932808876,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 2020
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1484.7604675292969,
      "epoch": 0.7761881901104176,
      "grad_norm": 0.265625,
      "learning_rate": 2.3031153676734365e-07,
      "loss": 0.1237,
      "num_tokens": 347002098.0,
      "reward": 0.596354179084301,
      "reward_std": 0.3074696436524391,
      "rewards/accuracy_reward": 0.14583333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 2021
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.8021240234375,
      "epoch": 0.7765722515602497,
      "grad_norm": 0.1953125,
      "learning_rate": 2.2988699771970385e-07,
      "loss": 0.1227,
      "num_tokens": 347179887.0,
      "reward": 0.4791666716337204,
      "reward_std": 0.2046803217381239,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 2022
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.4271240234375,
      "epoch": 0.7769563130100816,
      "grad_norm": 0.2109375,
      "learning_rate": 2.2946303468099e-07,
      "loss": 0.074,
      "num_tokens": 347340936.0,
      "reward": 0.567708358168602,
      "reward_std": 0.27322692796587944,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583358168602,
      "step": 2023
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1798.3229675292969,
      "epoch": 0.7773403744599136,
      "grad_norm": 0.185546875,
      "learning_rate": 2.2903964841407784e-07,
      "loss": 0.1045,
      "num_tokens": 347528751.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.22971466183662415,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515624925494194,
      "step": 2024
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.5000610351562,
      "epoch": 0.7777244359097456,
      "grad_norm": 0.224609375,
      "learning_rate": 2.2861683968080563e-07,
      "loss": 0.1282,
      "num_tokens": 347710375.0,
      "reward": 0.4244791865348816,
      "reward_std": 0.1733894031494856,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2025
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.6042175292969,
      "epoch": 0.7781084973595775,
      "grad_norm": 0.2734375,
      "learning_rate": 2.2819460924197227e-07,
      "loss": 0.1044,
      "num_tokens": 347876401.0,
      "reward": 0.4921875223517418,
      "reward_std": 0.2608902771025896,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541567325592,
      "step": 2026
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1471.7500305175781,
      "epoch": 0.7784925588094095,
      "grad_norm": 0.2412109375,
      "learning_rate": 2.2777295785733571e-07,
      "loss": 0.1237,
      "num_tokens": 348033249.0,
      "reward": 0.5546875149011612,
      "reward_std": 0.240811787545681,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 2027
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1433.4896240234375,
      "epoch": 0.7788766202592414,
      "grad_norm": 0.2265625,
      "learning_rate": 2.2735188628561267e-07,
      "loss": 0.1001,
      "num_tokens": 348184128.0,
      "reward": 0.5937500149011612,
      "reward_std": 0.31470008566975594,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333432674408,
      "step": 2028
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.9375305175781,
      "epoch": 0.7792606817090735,
      "grad_norm": 0.2236328125,
      "learning_rate": 2.2693139528447597e-07,
      "loss": 0.139,
      "num_tokens": 348360538.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.2814309224486351,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 2029
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.9791870117188,
      "epoch": 0.7796447431589054,
      "grad_norm": 0.2001953125,
      "learning_rate": 2.2651148561055397e-07,
      "loss": 0.0971,
      "num_tokens": 348523000.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.16264292411506176,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125074505806,
      "step": 2030
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.9271545410156,
      "epoch": 0.7800288046087374,
      "grad_norm": 0.2216796875,
      "learning_rate": 2.260921580194288e-07,
      "loss": 0.1368,
      "num_tokens": 348701505.0,
      "reward": 0.4713541865348816,
      "reward_std": 0.25395844131708145,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 2031
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.7500305175781,
      "epoch": 0.7804128660585694,
      "grad_norm": 0.205078125,
      "learning_rate": 2.2567341326563575e-07,
      "loss": 0.1239,
      "num_tokens": 348881945.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.20281335338950157,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 2032
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.6563110351562,
      "epoch": 0.7807969275084014,
      "grad_norm": 0.22265625,
      "learning_rate": 2.252552521026607e-07,
      "loss": 0.1118,
      "num_tokens": 349038720.0,
      "reward": 0.5130208507180214,
      "reward_std": 0.20367871224880219,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208432674408,
      "step": 2033
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1578.6666870117188,
      "epoch": 0.7811809889582333,
      "grad_norm": 0.244140625,
      "learning_rate": 2.248376752829398e-07,
      "loss": 0.0911,
      "num_tokens": 349206704.0,
      "reward": 0.471354179084301,
      "reward_std": 0.2142469324171543,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 2034
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.8542175292969,
      "epoch": 0.7815650504080653,
      "grad_norm": 0.2412109375,
      "learning_rate": 2.2442068355785747e-07,
      "loss": 0.1118,
      "num_tokens": 349369626.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.199974212795496,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 2035
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.8750305175781,
      "epoch": 0.7819491118578973,
      "grad_norm": 0.212890625,
      "learning_rate": 2.2400427767774576e-07,
      "loss": 0.103,
      "num_tokens": 349555206.0,
      "reward": 0.401041679084301,
      "reward_std": 0.16485381498932838,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 2036
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.2292175292969,
      "epoch": 0.7823331733077292,
      "grad_norm": 0.23046875,
      "learning_rate": 2.2358845839188212e-07,
      "loss": 0.0807,
      "num_tokens": 349722180.0,
      "reward": 0.502604179084301,
      "reward_std": 0.18768301606178284,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 2037
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1699.1042175292969,
      "epoch": 0.7827172347575612,
      "grad_norm": 0.23828125,
      "learning_rate": 2.2317322644848863e-07,
      "loss": 0.1096,
      "num_tokens": 349899678.0,
      "reward": 0.4635416716337204,
      "reward_std": 0.26764561235904694,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 2038
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1567.3125305175781,
      "epoch": 0.7831012962073932,
      "grad_norm": 0.251953125,
      "learning_rate": 2.227585825947304e-07,
      "loss": 0.0685,
      "num_tokens": 350066100.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.18613594956696033,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2039
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.7500610351562,
      "epoch": 0.7834853576572252,
      "grad_norm": 0.208984375,
      "learning_rate": 2.2234452757671475e-07,
      "loss": 0.0811,
      "num_tokens": 350248756.0,
      "reward": 0.3854166716337204,
      "reward_std": 0.16263555455952883,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 2040
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1580.0834045410156,
      "epoch": 0.7838694191070571,
      "grad_norm": 0.2041015625,
      "learning_rate": 2.2193106213948903e-07,
      "loss": 0.0935,
      "num_tokens": 350412596.0,
      "reward": 0.575520858168602,
      "reward_std": 0.27075278013944626,
      "rewards/accuracy_reward": 0.12500000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 2041
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1373.9062957763672,
      "epoch": 0.7842534805568891,
      "grad_norm": 0.232421875,
      "learning_rate": 2.2151818702703962e-07,
      "loss": 0.0841,
      "num_tokens": 350557979.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.211241252720356,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541641831398,
      "step": 2042
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.0625305175781,
      "epoch": 0.7846375420067211,
      "grad_norm": 0.1845703125,
      "learning_rate": 2.211059029822913e-07,
      "loss": 0.1006,
      "num_tokens": 350750473.0,
      "reward": 0.3906250074505806,
      "reward_std": 0.22560912324115634,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 2043
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.7812805175781,
      "epoch": 0.785021603456553,
      "grad_norm": 0.244140625,
      "learning_rate": 2.2069421074710466e-07,
      "loss": 0.0976,
      "num_tokens": 350913612.0,
      "reward": 0.5338541865348816,
      "reward_std": 0.2350412681698799,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875074505806,
      "step": 2044
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.7083740234375,
      "epoch": 0.785405664906385,
      "grad_norm": 0.2119140625,
      "learning_rate": 2.2028311106227555e-07,
      "loss": 0.1114,
      "num_tokens": 351079008.0,
      "reward": 0.4895833358168602,
      "reward_std": 0.21374059468507767,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2045
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1405.0000305175781,
      "epoch": 0.785789726356217,
      "grad_norm": 0.2470703125,
      "learning_rate": 2.1987260466753358e-07,
      "loss": 0.0753,
      "num_tokens": 351227704.0,
      "reward": 0.5937500149011612,
      "reward_std": 0.24058460257947445,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333283662796,
      "step": 2046
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.5,
      "epoch": 0.786173787806049,
      "grad_norm": 0.2158203125,
      "learning_rate": 2.1946269230154114e-07,
      "loss": 0.0947,
      "num_tokens": 351398088.0,
      "reward": 0.479166679084301,
      "reward_std": 0.17784428875893354,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 2047
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.9479675292969,
      "epoch": 0.7865578492558809,
      "grad_norm": 0.2421875,
      "learning_rate": 2.1905337470189134e-07,
      "loss": 0.0899,
      "num_tokens": 351573011.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.18797563575208187,
      "rewards/accuracy_reward": 0.03645833395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 2048
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.7708740234375,
      "epoch": 0.786941910705713,
      "grad_norm": 0.244140625,
      "learning_rate": 2.186446526051071e-07,
      "loss": 0.054,
      "num_tokens": 351747029.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.23509138077497482,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 2049
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.6354675292969,
      "epoch": 0.7873259721555449,
      "grad_norm": 0.2099609375,
      "learning_rate": 2.1823652674663982e-07,
      "loss": 0.1152,
      "num_tokens": 351921442.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.24363813549280167,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2050
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1693.4479675292969,
      "epoch": 0.7877100336053768,
      "grad_norm": 0.197265625,
      "learning_rate": 2.1782899786086836e-07,
      "loss": 0.0961,
      "num_tokens": 352098005.0,
      "reward": 0.466145858168602,
      "reward_std": 0.21279280073940754,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 2051
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1728.9479675292969,
      "epoch": 0.7880940950552089,
      "grad_norm": 0.1923828125,
      "learning_rate": 2.1742206668109697e-07,
      "loss": 0.1101,
      "num_tokens": 352276928.0,
      "reward": 0.5182291716337204,
      "reward_std": 0.2605302035808563,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 2052
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.7812805175781,
      "epoch": 0.7884781565050408,
      "grad_norm": 0.1728515625,
      "learning_rate": 2.1701573393955457e-07,
      "loss": 0.0616,
      "num_tokens": 352462331.0,
      "reward": 0.4687500074505806,
      "reward_std": 0.18477342557162046,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 2053
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1690.0312805175781,
      "epoch": 0.7888622179548728,
      "grad_norm": 0.240234375,
      "learning_rate": 2.1661000036739313e-07,
      "loss": 0.1227,
      "num_tokens": 352640310.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.21560904756188393,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.419270820915699,
      "step": 2054
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.6979370117188,
      "epoch": 0.7892462794047047,
      "grad_norm": 0.2373046875,
      "learning_rate": 2.162048666946866e-07,
      "loss": 0.1488,
      "num_tokens": 352817673.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.2192169614136219,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 2055
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1843.6354675292969,
      "epoch": 0.7896303408545368,
      "grad_norm": 0.16796875,
      "learning_rate": 2.1580033365042988e-07,
      "loss": 0.0895,
      "num_tokens": 353008070.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.24019308760762215,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 2056
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.6145935058594,
      "epoch": 0.7900144023043687,
      "grad_norm": 0.2275390625,
      "learning_rate": 2.1539640196253606e-07,
      "loss": 0.1215,
      "num_tokens": 353176361.0,
      "reward": 0.5625000074505806,
      "reward_std": 0.29572694562375546,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2057
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1515.5000305175781,
      "epoch": 0.7903984637542006,
      "grad_norm": 0.27734375,
      "learning_rate": 2.1499307235783714e-07,
      "loss": 0.1383,
      "num_tokens": 353336433.0,
      "reward": 0.5963541865348816,
      "reward_std": 0.2592284269630909,
      "rewards/accuracy_reward": 0.15624999534338713,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041716337204,
      "step": 2058
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1429.9791870117188,
      "epoch": 0.7907825252040327,
      "grad_norm": 0.25,
      "learning_rate": 2.1459034556208132e-07,
      "loss": 0.124,
      "num_tokens": 353489455.0,
      "reward": 0.6692708507180214,
      "reward_std": 0.2849585823714733,
      "rewards/accuracy_reward": 0.20833333861082792,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375074505806,
      "step": 2059
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1670.6875305175781,
      "epoch": 0.7911665866538646,
      "grad_norm": 0.2001953125,
      "learning_rate": 2.1418822229993222e-07,
      "loss": 0.0911,
      "num_tokens": 353662097.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.21099985763430595,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2060
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1498.9166870117188,
      "epoch": 0.7915506481036966,
      "grad_norm": 0.283203125,
      "learning_rate": 2.1378670329496714e-07,
      "loss": 0.1022,
      "num_tokens": 353819105.0,
      "reward": 0.5078125223517418,
      "reward_std": 0.17822469025850296,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 2061
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1634.5104370117188,
      "epoch": 0.7919347095535285,
      "grad_norm": 0.224609375,
      "learning_rate": 2.1338578926967671e-07,
      "loss": 0.0747,
      "num_tokens": 353991666.0,
      "reward": 0.5703125223517418,
      "reward_std": 0.2735128062777221,
      "rewards/accuracy_reward": 0.16666666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.403645820915699,
      "step": 2062
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.8125305175781,
      "epoch": 0.7923187710033606,
      "grad_norm": 0.22265625,
      "learning_rate": 2.1298548094546243e-07,
      "loss": 0.0929,
      "num_tokens": 354148056.0,
      "reward": 0.5416666716337204,
      "reward_std": 0.2103483434766531,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 2063
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.8437805175781,
      "epoch": 0.7927028324531925,
      "grad_norm": 0.18359375,
      "learning_rate": 2.125857790426361e-07,
      "loss": 0.096,
      "num_tokens": 354336617.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.1541914246045053,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 2064
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1709.3333892822266,
      "epoch": 0.7930868939030244,
      "grad_norm": 0.2314453125,
      "learning_rate": 2.1218668428041816e-07,
      "loss": 0.0885,
      "num_tokens": 354514001.0,
      "reward": 0.4895833507180214,
      "reward_std": 0.22726888488978148,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 2065
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.1042175292969,
      "epoch": 0.7934709553528565,
      "grad_norm": 0.20703125,
      "learning_rate": 2.1178819737693704e-07,
      "loss": 0.0804,
      "num_tokens": 354686227.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.25430573895573616,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.403645820915699,
      "step": 2066
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1347.875015258789,
      "epoch": 0.7938550168026884,
      "grad_norm": 0.23046875,
      "learning_rate": 2.1139031904922686e-07,
      "loss": 0.0278,
      "num_tokens": 354829543.0,
      "reward": 0.6067708507180214,
      "reward_std": 0.2694455534219742,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4817708358168602,
      "step": 2067
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.4896240234375,
      "epoch": 0.7942390782525204,
      "grad_norm": 0.2109375,
      "learning_rate": 2.1099305001322699e-07,
      "loss": 0.1432,
      "num_tokens": 354983462.0,
      "reward": 0.5833333432674408,
      "reward_std": 0.25564260594546795,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687500074505806,
      "step": 2068
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1516.0625610351562,
      "epoch": 0.7946231397023523,
      "grad_norm": 0.228515625,
      "learning_rate": 2.1059639098378012e-07,
      "loss": 0.1504,
      "num_tokens": 355142956.0,
      "reward": 0.541666679084301,
      "reward_std": 0.255469486117363,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4791666716337204,
      "step": 2069
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1710.0625610351562,
      "epoch": 0.7950072011521844,
      "grad_norm": 0.2060546875,
      "learning_rate": 2.1020034267463169e-07,
      "loss": 0.0977,
      "num_tokens": 355322962.0,
      "reward": 0.4687500074505806,
      "reward_std": 0.2349631767719984,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 2070
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1623.5521240234375,
      "epoch": 0.7953912626020163,
      "grad_norm": 0.2373046875,
      "learning_rate": 2.098049057984283e-07,
      "loss": 0.1212,
      "num_tokens": 355493759.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.26480118185281754,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 2071
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1844.4479370117188,
      "epoch": 0.7957753240518483,
      "grad_norm": 0.189453125,
      "learning_rate": 2.094100810667156e-07,
      "loss": 0.1075,
      "num_tokens": 355687466.0,
      "reward": 0.3802083432674408,
      "reward_std": 0.14950561430305243,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 2072
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1269.9167175292969,
      "epoch": 0.7961593855016803,
      "grad_norm": 0.25390625,
      "learning_rate": 2.090158691899385e-07,
      "loss": 0.1243,
      "num_tokens": 355823362.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.23769032210111618,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5104166567325592,
      "step": 2073
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1487.3854675292969,
      "epoch": 0.7965434469515122,
      "grad_norm": 0.267578125,
      "learning_rate": 2.0862227087743884e-07,
      "loss": 0.0801,
      "num_tokens": 355979447.0,
      "reward": 0.5651041865348816,
      "reward_std": 0.26508111134171486,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 2074
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1389.3229675292969,
      "epoch": 0.7969275084013442,
      "grad_norm": 0.259765625,
      "learning_rate": 2.0822928683745466e-07,
      "loss": 0.1607,
      "num_tokens": 356126478.0,
      "reward": 0.6562500149011612,
      "reward_std": 0.28249726444482803,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4895833432674408,
      "step": 2075
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.4896545410156,
      "epoch": 0.7973115698511762,
      "grad_norm": 0.2392578125,
      "learning_rate": 2.0783691777711808e-07,
      "loss": 0.1034,
      "num_tokens": 356304053.0,
      "reward": 0.4531250074505806,
      "reward_std": 0.20980257913470268,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 2076
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.9687805175781,
      "epoch": 0.7976956313010082,
      "grad_norm": 0.224609375,
      "learning_rate": 2.0744516440245535e-07,
      "loss": 0.1042,
      "num_tokens": 356474130.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.1938450038433075,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 2077
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1706.0000305175781,
      "epoch": 0.7980796927508401,
      "grad_norm": 0.1796875,
      "learning_rate": 2.0705402741838445e-07,
      "loss": 0.0698,
      "num_tokens": 356654602.0,
      "reward": 0.4557291716337204,
      "reward_std": 0.18503854423761368,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 2078
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.7500305175781,
      "epoch": 0.7984637542006721,
      "grad_norm": 0.2294921875,
      "learning_rate": 2.0666350752871433e-07,
      "loss": 0.1059,
      "num_tokens": 356824810.0,
      "reward": 0.463541679084301,
      "reward_std": 0.22835854813456535,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 2079
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.6563110351562,
      "epoch": 0.7988478156505041,
      "grad_norm": 0.2255859375,
      "learning_rate": 2.0627360543614336e-07,
      "loss": 0.0706,
      "num_tokens": 356987665.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.1994827277958393,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 2080
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1677.3021240234375,
      "epoch": 0.799231877100336,
      "grad_norm": 0.2109375,
      "learning_rate": 2.0588432184225874e-07,
      "loss": 0.1144,
      "num_tokens": 357163406.0,
      "reward": 0.5859375149011612,
      "reward_std": 0.34769829362630844,
      "rewards/accuracy_reward": 0.17708333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 2081
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1691.1875305175781,
      "epoch": 0.799615938550168,
      "grad_norm": 0.201171875,
      "learning_rate": 2.0549565744753428e-07,
      "loss": 0.0814,
      "num_tokens": 357338840.0,
      "reward": 0.4921875074505806,
      "reward_std": 0.19651136174798012,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 2082
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1378.0625457763672,
      "epoch": 0.8,
      "grad_norm": 0.2197265625,
      "learning_rate": 2.051076129513298e-07,
      "loss": 0.0761,
      "num_tokens": 357486006.0,
      "reward": 0.6302083358168602,
      "reward_std": 0.2621190268546343,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4947916716337204,
      "step": 2083
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1534.6875610351562,
      "epoch": 0.800384061449832,
      "grad_norm": 0.2578125,
      "learning_rate": 2.0472018905188942e-07,
      "loss": 0.0814,
      "num_tokens": 357646936.0,
      "reward": 0.5885416939854622,
      "reward_std": 0.2704901248216629,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416716337204,
      "step": 2084
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1186.2187805175781,
      "epoch": 0.8007681228996639,
      "grad_norm": 0.265625,
      "learning_rate": 2.0433338644634108e-07,
      "loss": 0.0531,
      "num_tokens": 357773397.0,
      "reward": 0.6614583507180214,
      "reward_std": 0.30322035402059555,
      "rewards/accuracy_reward": 0.16666666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4947916641831398,
      "step": 2085
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1644.9479675292969,
      "epoch": 0.8011521843494959,
      "grad_norm": 0.2294921875,
      "learning_rate": 2.0394720583069458e-07,
      "loss": 0.1438,
      "num_tokens": 357943176.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.2740132659673691,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 2086
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1838.5521240234375,
      "epoch": 0.8015362457993279,
      "grad_norm": 0.22265625,
      "learning_rate": 2.0356164789984e-07,
      "loss": 0.0786,
      "num_tokens": 358134557.0,
      "reward": 0.361979179084301,
      "reward_std": 0.13233829801902175,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 2087
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.1979370117188,
      "epoch": 0.8019203072491599,
      "grad_norm": 0.2392578125,
      "learning_rate": 2.0317671334754766e-07,
      "loss": 0.0387,
      "num_tokens": 358305328.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.2064055623486638,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 2088
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1740.1458740234375,
      "epoch": 0.8023043686989918,
      "grad_norm": 0.2138671875,
      "learning_rate": 2.0279240286646575e-07,
      "loss": 0.12,
      "num_tokens": 358485518.0,
      "reward": 0.481770858168602,
      "reward_std": 0.22999173030257225,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2089
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.0000305175781,
      "epoch": 0.8026884301488239,
      "grad_norm": 0.1923828125,
      "learning_rate": 2.0240871714811997e-07,
      "loss": 0.0999,
      "num_tokens": 358670150.0,
      "reward": 0.3776041865348816,
      "reward_std": 0.1103335446678102,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 2090
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.8229675292969,
      "epoch": 0.8030724915986558,
      "grad_norm": 0.2578125,
      "learning_rate": 2.02025656882911e-07,
      "loss": 0.0564,
      "num_tokens": 358838973.0,
      "reward": 0.5000000074505806,
      "reward_std": 0.22984366863965988,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 2091
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1606.0937805175781,
      "epoch": 0.8034565530484877,
      "grad_norm": 0.2216796875,
      "learning_rate": 2.0164322276011502e-07,
      "loss": 0.0989,
      "num_tokens": 359005398.0,
      "reward": 0.505208358168602,
      "reward_std": 0.226170277222991,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 2092
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.0104370117188,
      "epoch": 0.8038406144983198,
      "grad_norm": 0.23046875,
      "learning_rate": 2.0126141546788082e-07,
      "loss": 0.0913,
      "num_tokens": 359167175.0,
      "reward": 0.638020858168602,
      "reward_std": 0.30752505315467715,
      "rewards/accuracy_reward": 0.1822916679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375074505806,
      "step": 2093
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.0937805175781,
      "epoch": 0.8042246759481517,
      "grad_norm": 0.228515625,
      "learning_rate": 2.0088023569323e-07,
      "loss": 0.1297,
      "num_tokens": 359340232.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.26723663695156574,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 2094
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1849.1875305175781,
      "epoch": 0.8046087373979837,
      "grad_norm": 0.212890625,
      "learning_rate": 2.0049968412205403e-07,
      "loss": 0.1011,
      "num_tokens": 359534274.0,
      "reward": 0.3723958432674408,
      "reward_std": 0.19888461008667946,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 2095
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1684.5521240234375,
      "epoch": 0.8049927988478156,
      "grad_norm": 0.22265625,
      "learning_rate": 2.0011976143911488e-07,
      "loss": 0.1246,
      "num_tokens": 359710263.0,
      "reward": 0.5364583507180214,
      "reward_std": 0.29267745465040207,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 2096
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.8750305175781,
      "epoch": 0.8053768602976477,
      "grad_norm": 0.2373046875,
      "learning_rate": 1.9974046832804238e-07,
      "loss": 0.0768,
      "num_tokens": 359873299.0,
      "reward": 0.5052083358168602,
      "reward_std": 0.291251540184021,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 2097
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1581.7813110351562,
      "epoch": 0.8057609217474796,
      "grad_norm": 0.1962890625,
      "learning_rate": 1.9936180547133401e-07,
      "loss": 0.0665,
      "num_tokens": 360039102.0,
      "reward": 0.5937500074505806,
      "reward_std": 0.2048774864524603,
      "rewards/accuracy_reward": 0.14583333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166567325592,
      "step": 2098
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1670.0521240234375,
      "epoch": 0.8061449831973115,
      "grad_norm": 0.216796875,
      "learning_rate": 1.989837735503525e-07,
      "loss": 0.1322,
      "num_tokens": 360214635.0,
      "reward": 0.5546875,
      "reward_std": 0.26470642536878586,
      "rewards/accuracy_reward": 0.13541666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 2099
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.5313110351562,
      "epoch": 0.8065290446471436,
      "grad_norm": 0.2138671875,
      "learning_rate": 1.9860637324532576e-07,
      "loss": 0.0732,
      "num_tokens": 360396462.0,
      "reward": 0.4140625223517418,
      "reward_std": 0.14705593883991241,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 2100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.8959045410156,
      "epoch": 0.8069131060969755,
      "grad_norm": 0.232421875,
      "learning_rate": 1.9822960523534532e-07,
      "loss": 0.0801,
      "num_tokens": 360568692.0,
      "reward": 0.4791666716337204,
      "reward_std": 0.16258910298347473,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 2101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1827.1979675292969,
      "epoch": 0.8072971675468075,
      "grad_norm": 0.1865234375,
      "learning_rate": 1.978534701983642e-07,
      "loss": 0.0916,
      "num_tokens": 360756543.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.1887444704771042,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 2102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1447.1041870117188,
      "epoch": 0.8076812289966394,
      "grad_norm": 0.28125,
      "learning_rate": 1.9747796881119742e-07,
      "loss": 0.101,
      "num_tokens": 360911065.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.23519810661673546,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 2103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1721.0312805175781,
      "epoch": 0.8080652904464715,
      "grad_norm": 0.212890625,
      "learning_rate": 1.9710310174951895e-07,
      "loss": 0.0947,
      "num_tokens": 361091220.0,
      "reward": 0.4531250074505806,
      "reward_std": 0.22794055193662643,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 2104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1734.9271240234375,
      "epoch": 0.8084493518963034,
      "grad_norm": 0.2197265625,
      "learning_rate": 1.967288696878623e-07,
      "loss": 0.1224,
      "num_tokens": 361272941.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.2356172576546669,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 2105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.6875610351562,
      "epoch": 0.8088334133461353,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.9635527329961718e-07,
      "loss": 0.0601,
      "num_tokens": 361457103.0,
      "reward": 0.4166666716337204,
      "reward_std": 0.16116842534393072,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 2106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.5729675292969,
      "epoch": 0.8092174747959674,
      "grad_norm": 0.181640625,
      "learning_rate": 1.9598231325703057e-07,
      "loss": 0.1147,
      "num_tokens": 361644086.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.198367677628994,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 2107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1563.7292175292969,
      "epoch": 0.8096015362457993,
      "grad_norm": 0.2353515625,
      "learning_rate": 1.9560999023120366e-07,
      "loss": 0.0777,
      "num_tokens": 361808204.0,
      "reward": 0.4401041716337204,
      "reward_std": 0.1580662839114666,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208432674408,
      "step": 2108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.8646240234375,
      "epoch": 0.8099855976956313,
      "grad_norm": 0.2392578125,
      "learning_rate": 1.9523830489209214e-07,
      "loss": 0.1232,
      "num_tokens": 361974439.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.14716077595949173,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 2109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.5729370117188,
      "epoch": 0.8103696591454633,
      "grad_norm": 0.1953125,
      "learning_rate": 1.948672579085033e-07,
      "loss": 0.0847,
      "num_tokens": 362157710.0,
      "reward": 0.4140625149011612,
      "reward_std": 0.15566325932741165,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 2110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1502.1979675292969,
      "epoch": 0.8107537205952953,
      "grad_norm": 0.2080078125,
      "learning_rate": 1.944968499480966e-07,
      "loss": 0.1418,
      "num_tokens": 362315409.0,
      "reward": 0.4947916716337204,
      "reward_std": 0.1871498404070735,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 2111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1197.4271240234375,
      "epoch": 0.8111377820451272,
      "grad_norm": 0.306640625,
      "learning_rate": 1.9412708167738115e-07,
      "loss": 0.0638,
      "num_tokens": 362446506.0,
      "reward": 0.630208358168602,
      "reward_std": 0.27107882499694824,
      "rewards/accuracy_reward": 0.14583333022892475,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4843750074505806,
      "step": 2112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1679.6458740234375,
      "epoch": 0.8115218434949591,
      "grad_norm": 0.19140625,
      "learning_rate": 1.937579537617155e-07,
      "loss": 0.0592,
      "num_tokens": 362625448.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.15348825557157397,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 2113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.3333740234375,
      "epoch": 0.8119059049447912,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.9338946686530528e-07,
      "loss": 0.0537,
      "num_tokens": 362806224.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.20506209135055542,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 2114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.9375305175781,
      "epoch": 0.8122899663946231,
      "grad_norm": 0.22265625,
      "learning_rate": 1.930216216512031e-07,
      "loss": 0.1049,
      "num_tokens": 362964314.0,
      "reward": 0.5937500149011612,
      "reward_std": 0.2634131908416748,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 2115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1515.375,
      "epoch": 0.8126740278444551,
      "grad_norm": 0.224609375,
      "learning_rate": 1.9265441878130711e-07,
      "loss": 0.0876,
      "num_tokens": 363122822.0,
      "reward": 0.494791679084301,
      "reward_std": 0.20742330979555845,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 2116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1383.8854675292969,
      "epoch": 0.8130580892942871,
      "grad_norm": 0.25390625,
      "learning_rate": 1.9228785891635925e-07,
      "loss": 0.1522,
      "num_tokens": 363267859.0,
      "reward": 0.5572916716337204,
      "reward_std": 0.25899831391870975,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4843750074505806,
      "step": 2117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.2604370117188,
      "epoch": 0.8134421507441191,
      "grad_norm": 0.2431640625,
      "learning_rate": 1.9192194271594453e-07,
      "loss": 0.0819,
      "num_tokens": 363432436.0,
      "reward": 0.4817708432674408,
      "reward_std": 0.21047848835587502,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 2118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.1667175292969,
      "epoch": 0.813826212193951,
      "grad_norm": 0.220703125,
      "learning_rate": 1.9155667083848982e-07,
      "loss": 0.0893,
      "num_tokens": 363592220.0,
      "reward": 0.4921875298023224,
      "reward_std": 0.1967395767569542,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 2119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1670.0000305175781,
      "epoch": 0.814210273643783,
      "grad_norm": 0.21875,
      "learning_rate": 1.9119204394126275e-07,
      "loss": 0.0729,
      "num_tokens": 363764076.0,
      "reward": 0.440104179084301,
      "reward_std": 0.2003934569656849,
      "rewards/accuracy_reward": 0.0677083320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 2120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.4375305175781,
      "epoch": 0.814594335093615,
      "grad_norm": 0.189453125,
      "learning_rate": 1.9082806268036985e-07,
      "loss": 0.1525,
      "num_tokens": 363946798.0,
      "reward": 0.393229179084301,
      "reward_std": 0.18433028273284435,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791641831398,
      "step": 2121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.4791870117188,
      "epoch": 0.814978396543447,
      "grad_norm": 0.2109375,
      "learning_rate": 1.9046472771075667e-07,
      "loss": 0.0894,
      "num_tokens": 364122180.0,
      "reward": 0.486979179084301,
      "reward_std": 0.1311328075826168,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1433.9583740234375,
      "epoch": 0.8153624579932789,
      "grad_norm": 0.2294921875,
      "learning_rate": 1.9010203968620508e-07,
      "loss": 0.0885,
      "num_tokens": 364274976.0,
      "reward": 0.5000000223517418,
      "reward_std": 0.18817654624581337,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1720.3958435058594,
      "epoch": 0.8157465194431109,
      "grad_norm": 0.1748046875,
      "learning_rate": 1.8973999925933354e-07,
      "loss": 0.0475,
      "num_tokens": 364455270.0,
      "reward": 0.4765625223517418,
      "reward_std": 0.22846245393157005,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 2124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.7500305175781,
      "epoch": 0.8161305808929429,
      "grad_norm": 0.193359375,
      "learning_rate": 1.8937860708159444e-07,
      "loss": 0.0882,
      "num_tokens": 364636846.0,
      "reward": 0.3984375074505806,
      "reward_std": 0.16131010092794895,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708283662796,
      "step": 2125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.0000305175781,
      "epoch": 0.8165146423427748,
      "grad_norm": 0.2080078125,
      "learning_rate": 1.890178638032745e-07,
      "loss": 0.1239,
      "num_tokens": 364814366.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.17201082780957222,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 2126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1516.8438110351562,
      "epoch": 0.8168987037926068,
      "grad_norm": 0.23828125,
      "learning_rate": 1.886577700734922e-07,
      "loss": 0.0916,
      "num_tokens": 364977255.0,
      "reward": 0.4661458507180214,
      "reward_std": 0.18694286048412323,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291716337204,
      "step": 2127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1568.0937805175781,
      "epoch": 0.8172827652424388,
      "grad_norm": 0.23046875,
      "learning_rate": 1.8829832654019783e-07,
      "loss": 0.0887,
      "num_tokens": 365141848.0,
      "reward": 0.5338541865348816,
      "reward_std": 0.24547139927744865,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041567325592,
      "step": 2128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.4896240234375,
      "epoch": 0.8176668266922708,
      "grad_norm": 0.197265625,
      "learning_rate": 1.8793953385017097e-07,
      "loss": 0.0929,
      "num_tokens": 365317711.0,
      "reward": 0.4401041716337204,
      "reward_std": 0.19462633784860373,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1860.0416870117188,
      "epoch": 0.8180508881421027,
      "grad_norm": 0.1689453125,
      "learning_rate": 1.875813926490206e-07,
      "loss": 0.0743,
      "num_tokens": 365512123.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.18757784273475409,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458358168602,
      "step": 2130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.9583740234375,
      "epoch": 0.8184349495919347,
      "grad_norm": 0.2421875,
      "learning_rate": 1.8722390358118356e-07,
      "loss": 0.0911,
      "num_tokens": 365680415.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.20091649144887924,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 2131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1665.5729370117188,
      "epoch": 0.8188190110417667,
      "grad_norm": 0.20703125,
      "learning_rate": 1.8686706728992276e-07,
      "loss": 0.0841,
      "num_tokens": 365854430.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.11892827786505222,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 2132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.6354675292969,
      "epoch": 0.8192030724915986,
      "grad_norm": 0.185546875,
      "learning_rate": 1.8651088441732683e-07,
      "loss": 0.0954,
      "num_tokens": 366031211.0,
      "reward": 0.4817708432674408,
      "reward_std": 0.22506572678685188,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.3437805175781,
      "epoch": 0.8195871339414307,
      "grad_norm": 0.203125,
      "learning_rate": 1.8615535560430833e-07,
      "loss": 0.1346,
      "num_tokens": 366201412.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.24310643877834082,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1467.2083587646484,
      "epoch": 0.8199711953912626,
      "grad_norm": 0.32421875,
      "learning_rate": 1.858004814906034e-07,
      "loss": 0.0806,
      "num_tokens": 366356552.0,
      "reward": 0.4895833507180214,
      "reward_std": 0.18998697772622108,
      "rewards/accuracy_reward": 0.0677083320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.2188110351562,
      "epoch": 0.8203552568410946,
      "grad_norm": 0.2294921875,
      "learning_rate": 1.8544626271476976e-07,
      "loss": 0.1126,
      "num_tokens": 366527357.0,
      "reward": 0.471354179084301,
      "reward_std": 0.1723575545474887,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041716337204,
      "step": 2136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.1458435058594,
      "epoch": 0.8207393182909265,
      "grad_norm": 0.20703125,
      "learning_rate": 1.8509269991418598e-07,
      "loss": 0.0945,
      "num_tokens": 366711483.0,
      "reward": 0.471354179084301,
      "reward_std": 0.23454379476606846,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.9896240234375,
      "epoch": 0.8211233797407586,
      "grad_norm": 0.234375,
      "learning_rate": 1.8473979372505016e-07,
      "loss": 0.1083,
      "num_tokens": 366880754.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.18969125300645828,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453124925494194,
      "step": 2138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.3333740234375,
      "epoch": 0.8215074411905905,
      "grad_norm": 0.1953125,
      "learning_rate": 1.8438754478237938e-07,
      "loss": 0.0641,
      "num_tokens": 367041706.0,
      "reward": 0.583333358168602,
      "reward_std": 0.2859126552939415,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166567325592,
      "step": 2139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.0833740234375,
      "epoch": 0.8218915026404224,
      "grad_norm": 0.1943359375,
      "learning_rate": 1.840359537200075e-07,
      "loss": 0.0753,
      "num_tokens": 367223322.0,
      "reward": 0.442708358168602,
      "reward_std": 0.17645717738196254,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583283662796,
      "step": 2140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.5729675292969,
      "epoch": 0.8222755640902545,
      "grad_norm": 0.1767578125,
      "learning_rate": 1.8368502117058493e-07,
      "loss": 0.1043,
      "num_tokens": 367403281.0,
      "reward": 0.450520858168602,
      "reward_std": 0.20661207474768162,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 2141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1752.8854675292969,
      "epoch": 0.8226596255400864,
      "grad_norm": 0.2080078125,
      "learning_rate": 1.8333474776557678e-07,
      "loss": 0.145,
      "num_tokens": 367587974.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.2979853190481663,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 2142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.0625610351562,
      "epoch": 0.8230436869899184,
      "grad_norm": 0.2001953125,
      "learning_rate": 1.8298513413526296e-07,
      "loss": 0.1298,
      "num_tokens": 367771948.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.2631916031241417,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 2143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.0313110351562,
      "epoch": 0.8234277484397503,
      "grad_norm": 0.201171875,
      "learning_rate": 1.826361809087349e-07,
      "loss": 0.0834,
      "num_tokens": 367947023.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.1960831768810749,
      "rewards/accuracy_reward": 0.12500000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 2144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.9792175292969,
      "epoch": 0.8238118098895824,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.8228788871389677e-07,
      "loss": 0.1212,
      "num_tokens": 368122325.0,
      "reward": 0.4895833358168602,
      "reward_std": 0.20336461067199707,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1412.3437805175781,
      "epoch": 0.8241958713394143,
      "grad_norm": 0.251953125,
      "learning_rate": 1.819402581774631e-07,
      "loss": 0.1336,
      "num_tokens": 368270470.0,
      "reward": 0.677083358168602,
      "reward_std": 0.31960197165608406,
      "rewards/accuracy_reward": 0.18749999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4895833358168602,
      "step": 2146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1559.7917175292969,
      "epoch": 0.8245799327892462,
      "grad_norm": 0.22265625,
      "learning_rate": 1.815932899249574e-07,
      "loss": 0.0899,
      "num_tokens": 368434346.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.2078200951218605,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 2147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1616.4375610351562,
      "epoch": 0.8249639942390783,
      "grad_norm": 0.2197265625,
      "learning_rate": 1.8124698458071182e-07,
      "loss": 0.1104,
      "num_tokens": 368605876.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.2040056549012661,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1390.2604370117188,
      "epoch": 0.8253480556889102,
      "grad_norm": 0.27734375,
      "learning_rate": 1.809013427678654e-07,
      "loss": 0.1126,
      "num_tokens": 368754741.0,
      "reward": 0.494791679084301,
      "reward_std": 0.19674963876605034,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 2149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1420.8333740234375,
      "epoch": 0.8257321171387422,
      "grad_norm": 0.271484375,
      "learning_rate": 1.8055636510836375e-07,
      "loss": 0.1223,
      "num_tokens": 368907269.0,
      "reward": 0.6953125223517418,
      "reward_std": 0.3593708276748657,
      "rewards/accuracy_reward": 0.1979166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4973958283662796,
      "step": 2150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.4063110351562,
      "epoch": 0.8261161785885742,
      "grad_norm": 0.234375,
      "learning_rate": 1.8021205222295694e-07,
      "loss": 0.0974,
      "num_tokens": 369079212.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.15815974492579699,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 2151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.8333740234375,
      "epoch": 0.8265002400384062,
      "grad_norm": 0.2275390625,
      "learning_rate": 1.7986840473119885e-07,
      "loss": 0.0553,
      "num_tokens": 369241340.0,
      "reward": 0.541666679084301,
      "reward_std": 0.24485089629888535,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 2152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1646.3750610351562,
      "epoch": 0.8268843014882381,
      "grad_norm": 0.2412109375,
      "learning_rate": 1.7952542325144627e-07,
      "loss": 0.0866,
      "num_tokens": 369414424.0,
      "reward": 0.4348958358168602,
      "reward_std": 0.13998890295624733,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 2153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.7813110351562,
      "epoch": 0.82726836293807,
      "grad_norm": 0.216796875,
      "learning_rate": 1.791831084008575e-07,
      "loss": 0.1034,
      "num_tokens": 369592835.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.20869312435388565,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 2154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1775.4167175292969,
      "epoch": 0.8276524243879021,
      "grad_norm": 0.1875,
      "learning_rate": 1.7884146079539142e-07,
      "loss": 0.0737,
      "num_tokens": 369777283.0,
      "reward": 0.4635416865348816,
      "reward_std": 0.2296892274171114,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 2155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.2083740234375,
      "epoch": 0.828036485837734,
      "grad_norm": 0.2490234375,
      "learning_rate": 1.7850048104980598e-07,
      "loss": 0.0649,
      "num_tokens": 369955055.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.19050636515021324,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 2156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1659.9167175292969,
      "epoch": 0.828420547287566,
      "grad_norm": 0.2197265625,
      "learning_rate": 1.7816016977765751e-07,
      "loss": 0.0844,
      "num_tokens": 370129951.0,
      "reward": 0.502604179084301,
      "reward_std": 0.2321035135537386,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 2157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.7396545410156,
      "epoch": 0.828804608737398,
      "grad_norm": 0.2333984375,
      "learning_rate": 1.7782052759129988e-07,
      "loss": 0.1434,
      "num_tokens": 370312438.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.2460308950394392,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 2158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1645.8750610351562,
      "epoch": 0.82918867018723,
      "grad_norm": 0.2373046875,
      "learning_rate": 1.7748155510188245e-07,
      "loss": 0.1443,
      "num_tokens": 370485370.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.18555991351604462,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906250074505806,
      "step": 2159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.0834045410156,
      "epoch": 0.8295727316370619,
      "grad_norm": 0.2216796875,
      "learning_rate": 1.7714325291934966e-07,
      "loss": 0.1403,
      "num_tokens": 370654370.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.16299579106271267,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 2160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.6667175292969,
      "epoch": 0.8299567930868939,
      "grad_norm": 0.23828125,
      "learning_rate": 1.7680562165244015e-07,
      "loss": 0.09,
      "num_tokens": 370823058.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.18945418670773506,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 2161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1478.5729370117188,
      "epoch": 0.8303408545367259,
      "grad_norm": 0.265625,
      "learning_rate": 1.7646866190868504e-07,
      "loss": 0.1341,
      "num_tokens": 370980249.0,
      "reward": 0.5390625149011612,
      "reward_std": 0.25976698845624924,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291641831398,
      "step": 2162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1742.0625610351562,
      "epoch": 0.8307249159865578,
      "grad_norm": 0.1826171875,
      "learning_rate": 1.7613237429440704e-07,
      "loss": 0.0628,
      "num_tokens": 371162103.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.17907468415796757,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 2163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1492.5938110351562,
      "epoch": 0.8311089774363898,
      "grad_norm": 0.23828125,
      "learning_rate": 1.7579675941471945e-07,
      "loss": 0.1053,
      "num_tokens": 371320000.0,
      "reward": 0.5781250149011612,
      "reward_std": 0.25754209235310555,
      "rewards/accuracy_reward": 0.13541666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 2164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.0104675292969,
      "epoch": 0.8314930388862218,
      "grad_norm": 0.2001953125,
      "learning_rate": 1.7546181787352545e-07,
      "loss": 0.0553,
      "num_tokens": 371489481.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.20012061670422554,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 2165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.4167175292969,
      "epoch": 0.8318771003360538,
      "grad_norm": 0.2392578125,
      "learning_rate": 1.7512755027351622e-07,
      "loss": 0.0528,
      "num_tokens": 371654905.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.17948628775775433,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 2166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1467.7292175292969,
      "epoch": 0.8322611617858857,
      "grad_norm": 0.2734375,
      "learning_rate": 1.747939572161703e-07,
      "loss": 0.153,
      "num_tokens": 371810519.0,
      "reward": 0.5260416865348816,
      "reward_std": 0.25806440971791744,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 2167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1494.1563110351562,
      "epoch": 0.8326452232357178,
      "grad_norm": 0.2294921875,
      "learning_rate": 1.7446103930175244e-07,
      "loss": 0.1124,
      "num_tokens": 371967742.0,
      "reward": 0.5859375149011612,
      "reward_std": 0.22416381537914276,
      "rewards/accuracy_reward": 0.15624999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 2168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.7396240234375,
      "epoch": 0.8330292846855497,
      "grad_norm": 0.18359375,
      "learning_rate": 1.7412879712931288e-07,
      "loss": 0.0715,
      "num_tokens": 372144597.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.14126006793230772,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906250074505806,
      "step": 2169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1805.4584045410156,
      "epoch": 0.8334133461353816,
      "grad_norm": 0.1806640625,
      "learning_rate": 1.7379723129668554e-07,
      "loss": 0.094,
      "num_tokens": 372332209.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.19865353871136904,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 2170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.0521240234375,
      "epoch": 0.8337974075852136,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.734663424004875e-07,
      "loss": 0.1107,
      "num_tokens": 372495342.0,
      "reward": 0.5781250149011612,
      "reward_std": 0.2813493423163891,
      "rewards/accuracy_reward": 0.12500000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 2171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1475.2396240234375,
      "epoch": 0.8341814690350456,
      "grad_norm": 0.26171875,
      "learning_rate": 1.7313613103611764e-07,
      "loss": 0.07,
      "num_tokens": 372650485.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.2101295106112957,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4947916641831398,
      "step": 2172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.6979370117188,
      "epoch": 0.8345655304848776,
      "grad_norm": 0.2021484375,
      "learning_rate": 1.7280659779775595e-07,
      "loss": 0.0776,
      "num_tokens": 372832512.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.15393469855189323,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 2173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.2917175292969,
      "epoch": 0.8349495919347095,
      "grad_norm": 0.2099609375,
      "learning_rate": 1.72477743278362e-07,
      "loss": 0.1249,
      "num_tokens": 373013796.0,
      "reward": 0.502604179084301,
      "reward_std": 0.23662631213665009,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 2174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.5729675292969,
      "epoch": 0.8353336533845416,
      "grad_norm": 0.216796875,
      "learning_rate": 1.7214956806967408e-07,
      "loss": 0.0874,
      "num_tokens": 373193363.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.13100560009479523,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906249925494194,
      "step": 2175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1609.1458740234375,
      "epoch": 0.8357177148343735,
      "grad_norm": 0.189453125,
      "learning_rate": 1.7182207276220833e-07,
      "loss": 0.0347,
      "num_tokens": 373361953.0,
      "reward": 0.455729179084301,
      "reward_std": 0.1693996861577034,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 2176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.3021240234375,
      "epoch": 0.8361017762842055,
      "grad_norm": 0.1845703125,
      "learning_rate": 1.7149525794525732e-07,
      "loss": 0.1172,
      "num_tokens": 373549494.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.2854732424020767,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 2177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.1771240234375,
      "epoch": 0.8364858377340374,
      "grad_norm": 0.20703125,
      "learning_rate": 1.7116912420688908e-07,
      "loss": 0.1065,
      "num_tokens": 373740855.0,
      "reward": 0.463541679084301,
      "reward_std": 0.2467045895755291,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 2178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1341.4792175292969,
      "epoch": 0.8368698991838694,
      "grad_norm": 0.318359375,
      "learning_rate": 1.7084367213394613e-07,
      "loss": 0.122,
      "num_tokens": 373882413.0,
      "reward": 0.5859375149011612,
      "reward_std": 0.30683866888284683,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921875074505806,
      "step": 2179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.8333740234375,
      "epoch": 0.8372539606337014,
      "grad_norm": 0.201171875,
      "learning_rate": 1.705189023120447e-07,
      "loss": 0.0584,
      "num_tokens": 374065021.0,
      "reward": 0.4609375074505806,
      "reward_std": 0.20551318116486073,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 2180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1712.7604675292969,
      "epoch": 0.8376380220835333,
      "grad_norm": 0.208984375,
      "learning_rate": 1.7019481532557304e-07,
      "loss": 0.1358,
      "num_tokens": 374240806.0,
      "reward": 0.408854179084301,
      "reward_std": 0.19691233336925507,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.8854370117188,
      "epoch": 0.8380220835333654,
      "grad_norm": 0.2001953125,
      "learning_rate": 1.6987141175769086e-07,
      "loss": 0.0777,
      "num_tokens": 374405211.0,
      "reward": 0.6484375223517418,
      "reward_std": 0.2917296886444092,
      "rewards/accuracy_reward": 0.22916667070239782,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 2182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1653.0625610351562,
      "epoch": 0.8384061449831973,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.6954869219032793e-07,
      "loss": 0.1099,
      "num_tokens": 374577169.0,
      "reward": 0.5598958432674408,
      "reward_std": 0.30463316664099693,
      "rewards/accuracy_reward": 0.1614583320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.1146545410156,
      "epoch": 0.8387902064330293,
      "grad_norm": 0.25,
      "learning_rate": 1.692266572041837e-07,
      "loss": 0.1113,
      "num_tokens": 374740644.0,
      "reward": 0.5364583432674408,
      "reward_std": 0.3322247490286827,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531249925494194,
      "step": 2184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.4895935058594,
      "epoch": 0.8391742678828612,
      "grad_norm": 0.2109375,
      "learning_rate": 1.689053073787254e-07,
      "loss": 0.0801,
      "num_tokens": 374906059.0,
      "reward": 0.5364583507180214,
      "reward_std": 0.20890666544437408,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 2185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1524.5938110351562,
      "epoch": 0.8395583293326933,
      "grad_norm": 0.275390625,
      "learning_rate": 1.6858464329218743e-07,
      "loss": 0.0885,
      "num_tokens": 375065580.0,
      "reward": 0.5677083432674408,
      "reward_std": 0.2390003800392151,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 2186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.6875305175781,
      "epoch": 0.8399423907825252,
      "grad_norm": 0.1982421875,
      "learning_rate": 1.682646655215702e-07,
      "loss": 0.0822,
      "num_tokens": 375234942.0,
      "reward": 0.4531250074505806,
      "reward_std": 0.21384356543421745,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 2187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.1458740234375,
      "epoch": 0.8403264522323571,
      "grad_norm": 0.25,
      "learning_rate": 1.6794537464263963e-07,
      "loss": 0.0952,
      "num_tokens": 375404332.0,
      "reward": 0.502604179084301,
      "reward_std": 0.20882618241012096,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.6459045410156,
      "epoch": 0.8407105136821892,
      "grad_norm": 0.2119140625,
      "learning_rate": 1.6762677122992517e-07,
      "loss": 0.0813,
      "num_tokens": 375597666.0,
      "reward": 0.3802083432674408,
      "reward_std": 0.16886101476848125,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 2189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.6041870117188,
      "epoch": 0.8410945751320211,
      "grad_norm": 0.240234375,
      "learning_rate": 1.673088558567192e-07,
      "loss": 0.0932,
      "num_tokens": 375769140.0,
      "reward": 0.5000000223517418,
      "reward_std": 0.2926306053996086,
      "rewards/accuracy_reward": 0.10416666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333432674408,
      "step": 2190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1559.666748046875,
      "epoch": 0.8414786365818531,
      "grad_norm": 0.2421875,
      "learning_rate": 1.6699162909507656e-07,
      "loss": 0.0957,
      "num_tokens": 375931076.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.25282442569732666,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 2191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1426.4688110351562,
      "epoch": 0.8418626980316851,
      "grad_norm": 0.2451171875,
      "learning_rate": 1.6667509151581259e-07,
      "loss": 0.0307,
      "num_tokens": 376080913.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.2418363969773054,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333283662796,
      "step": 2192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.3958740234375,
      "epoch": 0.8422467594815171,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.6635924368850257e-07,
      "loss": 0.1105,
      "num_tokens": 376246943.0,
      "reward": 0.5234375298023224,
      "reward_std": 0.29801442474126816,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 2193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1445.1250457763672,
      "epoch": 0.842630820931349,
      "grad_norm": 0.330078125,
      "learning_rate": 1.6604408618148067e-07,
      "loss": 0.1128,
      "num_tokens": 376397811.0,
      "reward": 0.6250000074505806,
      "reward_std": 0.2406691424548626,
      "rewards/accuracy_reward": 0.21874999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 2194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.9479370117188,
      "epoch": 0.8430148823811809,
      "grad_norm": 0.2314453125,
      "learning_rate": 1.6572961956183903e-07,
      "loss": 0.0581,
      "num_tokens": 376554270.0,
      "reward": 0.630208358168602,
      "reward_std": 0.1841757521033287,
      "rewards/accuracy_reward": 0.15625000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583358168602,
      "step": 2195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1664.9479675292969,
      "epoch": 0.843398943831013,
      "grad_norm": 0.1865234375,
      "learning_rate": 1.6541584439542655e-07,
      "loss": 0.1082,
      "num_tokens": 376729937.0,
      "reward": 0.3984375074505806,
      "reward_std": 0.11811101902276278,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 2196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1455.8645935058594,
      "epoch": 0.8437830052808449,
      "grad_norm": 0.240234375,
      "learning_rate": 1.6510276124684785e-07,
      "loss": 0.0943,
      "num_tokens": 376882332.0,
      "reward": 0.494791679084301,
      "reward_std": 0.17802822962403297,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583358168602,
      "step": 2197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1832.6041870117188,
      "epoch": 0.8441670667306769,
      "grad_norm": 0.1689453125,
      "learning_rate": 1.6479037067946222e-07,
      "loss": 0.0518,
      "num_tokens": 377072614.0,
      "reward": 0.4401041716337204,
      "reward_std": 0.1961641786620021,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541641831398,
      "step": 2198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.7916870117188,
      "epoch": 0.8445511281805089,
      "grad_norm": 0.185546875,
      "learning_rate": 1.6447867325538316e-07,
      "loss": 0.0589,
      "num_tokens": 377257570.0,
      "reward": 0.4505208358168602,
      "reward_std": 0.21626177057623863,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 2199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.9687805175781,
      "epoch": 0.8449351896303409,
      "grad_norm": 0.21484375,
      "learning_rate": 1.641676695354766e-07,
      "loss": 0.1036,
      "num_tokens": 377431855.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.2532266341149807,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4374999925494194,
      "step": 2200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1496.9479370117188,
      "epoch": 0.8453192510801728,
      "grad_norm": 0.326171875,
      "learning_rate": 1.6385736007936023e-07,
      "loss": 0.2095,
      "num_tokens": 377590090.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.2583100087940693,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 2201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1442.3541870117188,
      "epoch": 0.8457033125300047,
      "grad_norm": 0.263671875,
      "learning_rate": 1.6354774544540257e-07,
      "loss": 0.1254,
      "num_tokens": 377744468.0,
      "reward": 0.6197916716337204,
      "reward_std": 0.3483257107436657,
      "rewards/accuracy_reward": 0.14583333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583358168602,
      "step": 2202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1453.7604370117188,
      "epoch": 0.8460873739798368,
      "grad_norm": 0.240234375,
      "learning_rate": 1.6323882619072186e-07,
      "loss": 0.0539,
      "num_tokens": 377898717.0,
      "reward": 0.5390625149011612,
      "reward_std": 0.20991886407136917,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765624925494194,
      "step": 2203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1724.8333435058594,
      "epoch": 0.8464714354296687,
      "grad_norm": 0.1845703125,
      "learning_rate": 1.6293060287118506e-07,
      "loss": 0.1055,
      "num_tokens": 378079717.0,
      "reward": 0.4036458507180214,
      "reward_std": 0.18153156153857708,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 2204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.1458435058594,
      "epoch": 0.8468554968795007,
      "grad_norm": 0.23828125,
      "learning_rate": 1.6262307604140686e-07,
      "loss": 0.0728,
      "num_tokens": 378240907.0,
      "reward": 0.5807291865348816,
      "reward_std": 0.23960649967193604,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125074505806,
      "step": 2205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1815.9583435058594,
      "epoch": 0.8472395583293327,
      "grad_norm": 0.208984375,
      "learning_rate": 1.6231624625474892e-07,
      "loss": 0.1232,
      "num_tokens": 378430359.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.1483747698366642,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666716337204,
      "step": 2206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1703.8542175292969,
      "epoch": 0.8476236197791647,
      "grad_norm": 0.21875,
      "learning_rate": 1.6201011406331828e-07,
      "loss": 0.1425,
      "num_tokens": 378610625.0,
      "reward": 0.5364583432674408,
      "reward_std": 0.2923012860119343,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 2207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.6354675292969,
      "epoch": 0.8480076812289966,
      "grad_norm": 0.30078125,
      "learning_rate": 1.6170468001796682e-07,
      "loss": 0.06,
      "num_tokens": 378769670.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.16718663647770882,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 2208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.7917175292969,
      "epoch": 0.8483917426788287,
      "grad_norm": 0.228515625,
      "learning_rate": 1.613999446682904e-07,
      "loss": 0.0735,
      "num_tokens": 378939242.0,
      "reward": 0.5989583432674408,
      "reward_std": 0.229454655200243,
      "rewards/accuracy_reward": 0.16666666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 2209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1738.1354675292969,
      "epoch": 0.8487758041286606,
      "grad_norm": 0.2060546875,
      "learning_rate": 1.610959085626275e-07,
      "loss": 0.0681,
      "num_tokens": 379119015.0,
      "reward": 0.479166679084301,
      "reward_std": 0.2522054947912693,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 2210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.0521240234375,
      "epoch": 0.8491598655784925,
      "grad_norm": 0.24609375,
      "learning_rate": 1.6079257224805843e-07,
      "loss": 0.1002,
      "num_tokens": 379274652.0,
      "reward": 0.5598958432674408,
      "reward_std": 0.2302112616598606,
      "rewards/accuracy_reward": 0.11458333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453124925494194,
      "step": 2211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1461.4583740234375,
      "epoch": 0.8495439270283245,
      "grad_norm": 0.267578125,
      "learning_rate": 1.6048993627040419e-07,
      "loss": 0.0778,
      "num_tokens": 379429936.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.23983483016490936,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 2212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.1667175292969,
      "epoch": 0.8499279884781565,
      "grad_norm": 0.177734375,
      "learning_rate": 1.6018800117422566e-07,
      "loss": 0.092,
      "num_tokens": 379611728.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.20934258587658405,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833283662796,
      "step": 2213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1509.0313110351562,
      "epoch": 0.8503120499279885,
      "grad_norm": 0.2421875,
      "learning_rate": 1.5988676750282264e-07,
      "loss": 0.0414,
      "num_tokens": 379771363.0,
      "reward": 0.5468750074505806,
      "reward_std": 0.22114777565002441,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083283662796,
      "step": 2214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1789.8750305175781,
      "epoch": 0.8506961113778204,
      "grad_norm": 0.2021484375,
      "learning_rate": 1.5958623579823277e-07,
      "loss": 0.0898,
      "num_tokens": 379957927.0,
      "reward": 0.393229179084301,
      "reward_std": 0.15431475825607777,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 2215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.0833740234375,
      "epoch": 0.8510801728276525,
      "grad_norm": 0.208984375,
      "learning_rate": 1.5928640660123045e-07,
      "loss": 0.0856,
      "num_tokens": 380132799.0,
      "reward": 0.5546875149011612,
      "reward_std": 0.21630488149821758,
      "rewards/accuracy_reward": 0.15624999534338713,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.1875610351562,
      "epoch": 0.8514642342774844,
      "grad_norm": 0.2080078125,
      "learning_rate": 1.5898728045132605e-07,
      "loss": 0.1061,
      "num_tokens": 380304025.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.19152519386261702,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 2217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1736.5833740234375,
      "epoch": 0.8518482957273164,
      "grad_norm": 0.21484375,
      "learning_rate": 1.5868885788676505e-07,
      "loss": 0.0639,
      "num_tokens": 380484569.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.24519539903849363,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 2218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.2813110351562,
      "epoch": 0.8522323571771483,
      "grad_norm": 0.1923828125,
      "learning_rate": 1.583911394445267e-07,
      "loss": 0.1251,
      "num_tokens": 380659044.0,
      "reward": 0.4218750149011612,
      "reward_std": 0.19878218695521355,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 2219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.9792175292969,
      "epoch": 0.8526164186269803,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.5809412566032324e-07,
      "loss": 0.0928,
      "num_tokens": 380833610.0,
      "reward": 0.447916679084301,
      "reward_std": 0.19117533415555954,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 2220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1497.5417175292969,
      "epoch": 0.8530004800768123,
      "grad_norm": 0.259765625,
      "learning_rate": 1.5779781706859918e-07,
      "loss": 0.1697,
      "num_tokens": 380990998.0,
      "reward": 0.5911458507180214,
      "reward_std": 0.2519662007689476,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4869791567325592,
      "step": 2221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.7396545410156,
      "epoch": 0.8533845415266442,
      "grad_norm": 0.2138671875,
      "learning_rate": 1.5750221420252988e-07,
      "loss": 0.1542,
      "num_tokens": 381164637.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.21110089868307114,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 2222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1830.3542175292969,
      "epoch": 0.8537686029764763,
      "grad_norm": 0.201171875,
      "learning_rate": 1.5720731759402095e-07,
      "loss": 0.1128,
      "num_tokens": 381355839.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.2520427331328392,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 2223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.0937805175781,
      "epoch": 0.8541526644263082,
      "grad_norm": 0.271484375,
      "learning_rate": 1.5691312777370703e-07,
      "loss": 0.1446,
      "num_tokens": 381512600.0,
      "reward": 0.653645858168602,
      "reward_std": 0.27874375879764557,
      "rewards/accuracy_reward": 0.18749999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 2224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.4896240234375,
      "epoch": 0.8545367258761402,
      "grad_norm": 0.2490234375,
      "learning_rate": 1.5661964527095122e-07,
      "loss": 0.0515,
      "num_tokens": 381677375.0,
      "reward": 0.5286458432674408,
      "reward_std": 0.2914812322705984,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 2225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1714.4792175292969,
      "epoch": 0.8549207873259722,
      "grad_norm": 0.2099609375,
      "learning_rate": 1.5632687061384366e-07,
      "loss": 0.1504,
      "num_tokens": 381856093.0,
      "reward": 0.4713541865348816,
      "reward_std": 0.18894057162106037,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 2226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1627.6459045410156,
      "epoch": 0.8553048487758041,
      "grad_norm": 0.2412109375,
      "learning_rate": 1.5603480432920087e-07,
      "loss": 0.1357,
      "num_tokens": 382026859.0,
      "reward": 0.4140625074505806,
      "reward_std": 0.1512828003615141,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.0000610351562,
      "epoch": 0.8556889102256361,
      "grad_norm": 0.2392578125,
      "learning_rate": 1.5574344694256458e-07,
      "loss": 0.0944,
      "num_tokens": 382193899.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.2590879499912262,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 2228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.9791870117188,
      "epoch": 0.856072971675468,
      "grad_norm": 0.181640625,
      "learning_rate": 1.554527989782013e-07,
      "loss": 0.0922,
      "num_tokens": 382369249.0,
      "reward": 0.5416666865348816,
      "reward_std": 0.2137383222579956,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1377.8750610351562,
      "epoch": 0.8564570331253001,
      "grad_norm": 0.251953125,
      "learning_rate": 1.5516286095910072e-07,
      "loss": 0.1568,
      "num_tokens": 382514405.0,
      "reward": 0.5598958432674408,
      "reward_std": 0.21594425104558468,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 2230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1457.1979675292969,
      "epoch": 0.856841094575132,
      "grad_norm": 0.2353515625,
      "learning_rate": 1.548736334069751e-07,
      "loss": 0.1349,
      "num_tokens": 382667632.0,
      "reward": 0.5104166865348816,
      "reward_std": 0.22021548449993134,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 2231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1477.9063110351562,
      "epoch": 0.857225156024964,
      "grad_norm": 0.26171875,
      "learning_rate": 1.545851168422582e-07,
      "loss": 0.134,
      "num_tokens": 382826255.0,
      "reward": 0.6119791716337204,
      "reward_std": 0.3137410581111908,
      "rewards/accuracy_reward": 0.17708333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 2232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.7396240234375,
      "epoch": 0.857609217474796,
      "grad_norm": 0.23046875,
      "learning_rate": 1.5429731178410468e-07,
      "loss": 0.1121,
      "num_tokens": 382999158.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.1574382148683071,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 2233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.2604370117188,
      "epoch": 0.857993278924628,
      "grad_norm": 0.19921875,
      "learning_rate": 1.54010218750389e-07,
      "loss": 0.1105,
      "num_tokens": 383171215.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.11825977638363838,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 2234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.1562805175781,
      "epoch": 0.8583773403744599,
      "grad_norm": 0.1875,
      "learning_rate": 1.537238382577038e-07,
      "loss": 0.1048,
      "num_tokens": 383346886.0,
      "reward": 0.424479179084301,
      "reward_std": 0.14029468968510628,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 2235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.6979675292969,
      "epoch": 0.8587614018242918,
      "grad_norm": 0.240234375,
      "learning_rate": 1.5343817082136026e-07,
      "loss": 0.0731,
      "num_tokens": 383532633.0,
      "reward": 0.486979179084301,
      "reward_std": 0.17005395144224167,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.6771240234375,
      "epoch": 0.8591454632741239,
      "grad_norm": 0.2109375,
      "learning_rate": 1.5315321695538605e-07,
      "loss": 0.0761,
      "num_tokens": 383713298.0,
      "reward": 0.3697916716337204,
      "reward_std": 0.13926101848483086,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916641831398,
      "step": 2237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.1042175292969,
      "epoch": 0.8595295247239558,
      "grad_norm": 0.22265625,
      "learning_rate": 1.5286897717252527e-07,
      "loss": 0.1204,
      "num_tokens": 383878852.0,
      "reward": 0.6276041865348816,
      "reward_std": 0.3260718621313572,
      "rewards/accuracy_reward": 0.19791666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 2238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1715.6250305175781,
      "epoch": 0.8599135861737878,
      "grad_norm": 0.23046875,
      "learning_rate": 1.5258545198423644e-07,
      "loss": 0.1574,
      "num_tokens": 384058424.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.295002531260252,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1694.8958435058594,
      "epoch": 0.8602976476236198,
      "grad_norm": 0.1728515625,
      "learning_rate": 1.5230264190069302e-07,
      "loss": 0.0812,
      "num_tokens": 384236718.0,
      "reward": 0.4505208507180214,
      "reward_std": 0.12787536857649684,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1791.1354370117188,
      "epoch": 0.8606817090734518,
      "grad_norm": 0.166015625,
      "learning_rate": 1.5202054743078122e-07,
      "loss": 0.0813,
      "num_tokens": 384421099.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.19968359917402267,
      "rewards/accuracy_reward": 0.1354166716337204,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3411458283662796,
      "step": 2241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.9583740234375,
      "epoch": 0.8610657705232837,
      "grad_norm": 0.220703125,
      "learning_rate": 1.5173916908209966e-07,
      "loss": 0.0797,
      "num_tokens": 384584183.0,
      "reward": 0.5104166716337204,
      "reward_std": 0.20716453716158867,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1820.5417175292969,
      "epoch": 0.8614498319731158,
      "grad_norm": 0.189453125,
      "learning_rate": 1.5145850736095834e-07,
      "loss": 0.0663,
      "num_tokens": 384773155.0,
      "reward": 0.4062500074505806,
      "reward_std": 0.1768946833908558,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 2243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1385.8020935058594,
      "epoch": 0.8618338934229477,
      "grad_norm": 0.234375,
      "learning_rate": 1.51178562772378e-07,
      "loss": 0.1224,
      "num_tokens": 384918968.0,
      "reward": 0.6250000223517418,
      "reward_std": 0.23759765923023224,
      "rewards/accuracy_reward": 0.14583333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4791666641831398,
      "step": 2244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.9271240234375,
      "epoch": 0.8622179548727796,
      "grad_norm": 0.1591796875,
      "learning_rate": 1.5089933582008883e-07,
      "loss": 0.0628,
      "num_tokens": 385113521.0,
      "reward": 0.4895833432674408,
      "reward_std": 0.18525387346744537,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 2245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1783.1458740234375,
      "epoch": 0.8626020163226116,
      "grad_norm": 0.203125,
      "learning_rate": 1.5062082700652964e-07,
      "loss": 0.1096,
      "num_tokens": 385298159.0,
      "reward": 0.489583358168602,
      "reward_std": 0.2521274536848068,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 2246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1743.4583435058594,
      "epoch": 0.8629860777724436,
      "grad_norm": 0.181640625,
      "learning_rate": 1.5034303683284703e-07,
      "loss": 0.1188,
      "num_tokens": 385477867.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.23498260229825974,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1835.3958740234375,
      "epoch": 0.8633701392222756,
      "grad_norm": 0.17578125,
      "learning_rate": 1.500659657988946e-07,
      "loss": 0.0883,
      "num_tokens": 385669145.0,
      "reward": 0.3906250149011612,
      "reward_std": 0.20220431871712208,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 2248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.0625610351562,
      "epoch": 0.8637542006721075,
      "grad_norm": 0.193359375,
      "learning_rate": 1.4978961440323212e-07,
      "loss": 0.0753,
      "num_tokens": 385840167.0,
      "reward": 0.5156250223517418,
      "reward_std": 0.17419143579900265,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 2249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1576.5729675292969,
      "epoch": 0.8641382621219396,
      "grad_norm": 0.251953125,
      "learning_rate": 1.495139831431238e-07,
      "loss": 0.137,
      "num_tokens": 386005830.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.22227400541305542,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 2250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1549.2812805175781,
      "epoch": 0.8645223235717715,
      "grad_norm": 0.234375,
      "learning_rate": 1.4923907251453877e-07,
      "loss": 0.088,
      "num_tokens": 386167729.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.13730287551879883,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 2251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1709.1562805175781,
      "epoch": 0.8649063850216034,
      "grad_norm": 0.20703125,
      "learning_rate": 1.4896488301214887e-07,
      "loss": 0.0792,
      "num_tokens": 386347776.0,
      "reward": 0.4453125223517418,
      "reward_std": 0.16071300953626633,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 2252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1510.6354675292969,
      "epoch": 0.8652904464714354,
      "grad_norm": 0.197265625,
      "learning_rate": 1.48691415129329e-07,
      "loss": 0.1192,
      "num_tokens": 386505285.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.17981746792793274,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1723.4375610351562,
      "epoch": 0.8656745079212674,
      "grad_norm": 0.21875,
      "learning_rate": 1.484186693581548e-07,
      "loss": 0.0614,
      "num_tokens": 386682991.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.1984549006447196,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 2254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.9479370117188,
      "epoch": 0.8660585693710994,
      "grad_norm": 0.2314453125,
      "learning_rate": 1.4814664618940326e-07,
      "loss": 0.0729,
      "num_tokens": 386865402.0,
      "reward": 0.3776041716337204,
      "reward_std": 0.1317969709634781,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 2255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1459.0521240234375,
      "epoch": 0.8664426308209313,
      "grad_norm": 0.26171875,
      "learning_rate": 1.4787534611255057e-07,
      "loss": 0.0745,
      "num_tokens": 387017815.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.22136707417666912,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 2256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1388.6979675292969,
      "epoch": 0.8668266922707634,
      "grad_norm": 0.21875,
      "learning_rate": 1.4760476961577237e-07,
      "loss": 0.0768,
      "num_tokens": 387164994.0,
      "reward": 0.7161458432674408,
      "reward_std": 0.2929651439189911,
      "rewards/accuracy_reward": 0.2395833283662796,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765624925494194,
      "step": 2257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.1667175292969,
      "epoch": 0.8672107537205953,
      "grad_norm": 0.203125,
      "learning_rate": 1.4733491718594166e-07,
      "loss": 0.0927,
      "num_tokens": 387335266.0,
      "reward": 0.5572916939854622,
      "reward_std": 0.27640724554657936,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 2258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.6042175292969,
      "epoch": 0.8675948151704272,
      "grad_norm": 0.259765625,
      "learning_rate": 1.4706578930862903e-07,
      "loss": 0.1247,
      "num_tokens": 387496964.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.28816304355859756,
      "rewards/accuracy_reward": 0.12500000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.4688110351562,
      "epoch": 0.8679788766202592,
      "grad_norm": 0.25,
      "learning_rate": 1.4679738646810115e-07,
      "loss": 0.1159,
      "num_tokens": 387658937.0,
      "reward": 0.4739583507180214,
      "reward_std": 0.22995496168732643,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 2260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1393.6667175292969,
      "epoch": 0.8683629380700912,
      "grad_norm": 0.236328125,
      "learning_rate": 1.4652970914732005e-07,
      "loss": 0.1201,
      "num_tokens": 387806673.0,
      "reward": 0.5651041865348816,
      "reward_std": 0.256274800747633,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 2261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.2187805175781,
      "epoch": 0.8687469995199232,
      "grad_norm": 0.255859375,
      "learning_rate": 1.4626275782794218e-07,
      "loss": 0.1025,
      "num_tokens": 387976438.0,
      "reward": 0.40625,
      "reward_std": 0.1859695054590702,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 2262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.875,
      "epoch": 0.8691310609697551,
      "grad_norm": 0.1982421875,
      "learning_rate": 1.459965329903178e-07,
      "loss": 0.0801,
      "num_tokens": 388144090.0,
      "reward": 0.5468750149011612,
      "reward_std": 0.2114749290049076,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 2263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1691.3854370117188,
      "epoch": 0.8695151224195872,
      "grad_norm": 0.205078125,
      "learning_rate": 1.457310351134901e-07,
      "loss": 0.0594,
      "num_tokens": 388320327.0,
      "reward": 0.4661458507180214,
      "reward_std": 0.19029607251286507,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 2264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1355.6667175292969,
      "epoch": 0.8698991838694191,
      "grad_norm": 0.1953125,
      "learning_rate": 1.454662646751936e-07,
      "loss": 0.0668,
      "num_tokens": 388463175.0,
      "reward": 0.6510416865348816,
      "reward_std": 0.23455711547285318,
      "rewards/accuracy_reward": 0.1979166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250074505806,
      "step": 2265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.1562805175781,
      "epoch": 0.870283245319251,
      "grad_norm": 0.2314453125,
      "learning_rate": 1.452022221518545e-07,
      "loss": 0.1239,
      "num_tokens": 388627054.0,
      "reward": 0.557291679084301,
      "reward_std": 0.24267536029219627,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 2266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1572.9062805175781,
      "epoch": 0.8706673067690831,
      "grad_norm": 0.1787109375,
      "learning_rate": 1.449389080185887e-07,
      "loss": 0.0834,
      "num_tokens": 388792581.0,
      "reward": 0.4869791716337204,
      "reward_std": 0.20101991295814514,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.424479179084301,
      "step": 2267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.8334045410156,
      "epoch": 0.871051368218915,
      "grad_norm": 0.212890625,
      "learning_rate": 1.44676322749202e-07,
      "loss": 0.106,
      "num_tokens": 388950789.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.1718832477927208,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 2268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1782.9375610351562,
      "epoch": 0.871435429668747,
      "grad_norm": 0.1728515625,
      "learning_rate": 1.44414466816188e-07,
      "loss": 0.1135,
      "num_tokens": 389140095.0,
      "reward": 0.3333333432674408,
      "reward_std": 0.08616935554891825,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333358168602,
      "step": 2269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1468.4271240234375,
      "epoch": 0.8718194911185789,
      "grad_norm": 0.2216796875,
      "learning_rate": 1.4415334069072852e-07,
      "loss": 0.1111,
      "num_tokens": 389295336.0,
      "reward": 0.5703125298023224,
      "reward_std": 0.2700529024004936,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.455729179084301,
      "step": 2270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1725.6250305175781,
      "epoch": 0.872203552568411,
      "grad_norm": 0.20703125,
      "learning_rate": 1.438929448426918e-07,
      "loss": 0.1286,
      "num_tokens": 389473060.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.2127217873930931,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 2271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1614.7917175292969,
      "epoch": 0.8725876140182429,
      "grad_norm": 0.21484375,
      "learning_rate": 1.4363327974063238e-07,
      "loss": 0.096,
      "num_tokens": 389643520.0,
      "reward": 0.408854179084301,
      "reward_std": 0.14899443043395877,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1478.2500457763672,
      "epoch": 0.8729716754680749,
      "grad_norm": 0.2333984375,
      "learning_rate": 1.433743458517894e-07,
      "loss": 0.0911,
      "num_tokens": 389797208.0,
      "reward": 0.6197916865348816,
      "reward_std": 0.3059423863887787,
      "rewards/accuracy_reward": 0.17708333767950535,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 2273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1774.2813110351562,
      "epoch": 0.8733557369179069,
      "grad_norm": 0.21875,
      "learning_rate": 1.4311614364208683e-07,
      "loss": 0.0416,
      "num_tokens": 389982579.0,
      "reward": 0.4322916716337204,
      "reward_std": 0.22357287257909775,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 2274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.1771240234375,
      "epoch": 0.8737397983677389,
      "grad_norm": 0.2353515625,
      "learning_rate": 1.428586735761315e-07,
      "loss": 0.0964,
      "num_tokens": 390159156.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.21173461340367794,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 2275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1687.7188110351562,
      "epoch": 0.8741238598175708,
      "grad_norm": 0.2451171875,
      "learning_rate": 1.426019361172135e-07,
      "loss": 0.111,
      "num_tokens": 390334673.0,
      "reward": 0.4973958358168602,
      "reward_std": 0.2846258878707886,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1542.4271240234375,
      "epoch": 0.8745079212674027,
      "grad_norm": 0.232421875,
      "learning_rate": 1.4234593172730387e-07,
      "loss": 0.1118,
      "num_tokens": 390497506.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.2026742547750473,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453124925494194,
      "step": 2277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1326.7083740234375,
      "epoch": 0.8748919827172348,
      "grad_norm": 0.2734375,
      "learning_rate": 1.420906608670551e-07,
      "loss": 0.0448,
      "num_tokens": 390638078.0,
      "reward": 0.6953125149011612,
      "reward_std": 0.3030703663825989,
      "rewards/accuracy_reward": 0.2083333283662796,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4869791716337204,
      "step": 2278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1539.3437805175781,
      "epoch": 0.8752760441670667,
      "grad_norm": 0.2431640625,
      "learning_rate": 1.4183612399579978e-07,
      "loss": 0.1499,
      "num_tokens": 390798327.0,
      "reward": 0.5520833432674408,
      "reward_std": 0.25431638583540916,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1561.5625305175781,
      "epoch": 0.8756601056168987,
      "grad_norm": 0.240234375,
      "learning_rate": 1.4158232157154962e-07,
      "loss": 0.101,
      "num_tokens": 390960893.0,
      "reward": 0.5520833507180214,
      "reward_std": 0.20226197689771652,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 2280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1470.0000610351562,
      "epoch": 0.8760441670667307,
      "grad_norm": 0.275390625,
      "learning_rate": 1.4132925405099472e-07,
      "loss": 0.0507,
      "num_tokens": 391117365.0,
      "reward": 0.486979179084301,
      "reward_std": 0.19865096919238567,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291641831398,
      "step": 2281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.0729675292969,
      "epoch": 0.8764282285165627,
      "grad_norm": 0.193359375,
      "learning_rate": 1.4107692188950277e-07,
      "loss": 0.0403,
      "num_tokens": 391299820.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.12205765303224325,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 2282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.5417175292969,
      "epoch": 0.8768122899663946,
      "grad_norm": 0.255859375,
      "learning_rate": 1.4082532554111867e-07,
      "loss": 0.1106,
      "num_tokens": 391477536.0,
      "reward": 0.5598958432674408,
      "reward_std": 0.33260276168584824,
      "rewards/accuracy_reward": 0.18749999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 2283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1503.4479522705078,
      "epoch": 0.8771963514162266,
      "grad_norm": 0.1943359375,
      "learning_rate": 1.4057446545856266e-07,
      "loss": 0.1139,
      "num_tokens": 391636051.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.16621900349855423,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 2284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1641.1250610351562,
      "epoch": 0.8775804128660586,
      "grad_norm": 0.212890625,
      "learning_rate": 1.403243420932307e-07,
      "loss": 0.1478,
      "num_tokens": 391808919.0,
      "reward": 0.5520833432674408,
      "reward_std": 0.29091629944741726,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 2285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.6875610351562,
      "epoch": 0.8779644743158905,
      "grad_norm": 0.251953125,
      "learning_rate": 1.4007495589519274e-07,
      "loss": 0.1323,
      "num_tokens": 391975857.0,
      "reward": 0.5208333656191826,
      "reward_std": 0.2632324993610382,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 2286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.7083740234375,
      "epoch": 0.8783485357657225,
      "grad_norm": 0.18359375,
      "learning_rate": 1.3982630731319268e-07,
      "loss": 0.0958,
      "num_tokens": 392136237.0,
      "reward": 0.502604179084301,
      "reward_std": 0.16341599076986313,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 2287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1648.4687805175781,
      "epoch": 0.8787325972155545,
      "grad_norm": 0.2294921875,
      "learning_rate": 1.3957839679464654e-07,
      "loss": 0.1556,
      "num_tokens": 392307586.0,
      "reward": 0.505208358168602,
      "reward_std": 0.22636481001973152,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 2288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1835.1458740234375,
      "epoch": 0.8791166586653865,
      "grad_norm": 0.1884765625,
      "learning_rate": 1.3933122478564292e-07,
      "loss": 0.103,
      "num_tokens": 392498112.0,
      "reward": 0.518229179084301,
      "reward_std": 0.3113022670149803,
      "rewards/accuracy_reward": 0.14583333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 2289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1517.0312805175781,
      "epoch": 0.8795007201152184,
      "grad_norm": 0.201171875,
      "learning_rate": 1.3908479173094111e-07,
      "loss": 0.1101,
      "num_tokens": 392660699.0,
      "reward": 0.5807291865348816,
      "reward_std": 0.22385356947779655,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453124925494194,
      "step": 2290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.7396240234375,
      "epoch": 0.8798847815650505,
      "grad_norm": 0.154296875,
      "learning_rate": 1.388390980739711e-07,
      "loss": 0.109,
      "num_tokens": 392850634.0,
      "reward": 0.3958333507180214,
      "reward_std": 0.1925726169720292,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3333333358168602,
      "step": 2291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1605.0000305175781,
      "epoch": 0.8802688430148824,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.3859414425683198e-07,
      "loss": 0.1163,
      "num_tokens": 393018474.0,
      "reward": 0.5208333656191826,
      "reward_std": 0.30329722817987204,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.5000305175781,
      "epoch": 0.8806529044647143,
      "grad_norm": 0.2275390625,
      "learning_rate": 1.3834993072029183e-07,
      "loss": 0.0861,
      "num_tokens": 393188674.0,
      "reward": 0.4843750074505806,
      "reward_std": 0.2187075838446617,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250074505806,
      "step": 2293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1620.2604370117188,
      "epoch": 0.8810369659145463,
      "grad_norm": 0.2216796875,
      "learning_rate": 1.3810645790378682e-07,
      "loss": 0.1264,
      "num_tokens": 393360027.0,
      "reward": 0.502604179084301,
      "reward_std": 0.2631384413689375,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 2294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1481.6458740234375,
      "epoch": 0.8814210273643783,
      "grad_norm": 0.232421875,
      "learning_rate": 1.3786372624542e-07,
      "loss": 0.0897,
      "num_tokens": 393517209.0,
      "reward": 0.5781250149011612,
      "reward_std": 0.1546234730631113,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 2295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.6563110351562,
      "epoch": 0.8818050888142103,
      "grad_norm": 0.1904296875,
      "learning_rate": 1.376217361819609e-07,
      "loss": 0.0876,
      "num_tokens": 393680592.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.128186016343534,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.1146240234375,
      "epoch": 0.8821891502640422,
      "grad_norm": 0.23828125,
      "learning_rate": 1.3738048814884455e-07,
      "loss": 0.1473,
      "num_tokens": 393842091.0,
      "reward": 0.4843750223517418,
      "reward_std": 0.2057260498404503,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.442708320915699,
      "step": 2297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1622.1042175292969,
      "epoch": 0.8825732117138743,
      "grad_norm": 0.21875,
      "learning_rate": 1.3713998258017094e-07,
      "loss": 0.1003,
      "num_tokens": 394009549.0,
      "reward": 0.575520858168602,
      "reward_std": 0.2729686573147774,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 2298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1342.0000305175781,
      "epoch": 0.8829572731637062,
      "grad_norm": 0.25390625,
      "learning_rate": 1.3690021990870392e-07,
      "loss": 0.0716,
      "num_tokens": 394150413.0,
      "reward": 0.6953125298023224,
      "reward_std": 0.29031578078866005,
      "rewards/accuracy_reward": 0.1979166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4973958432674408,
      "step": 2299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.0938110351562,
      "epoch": 0.8833413346135381,
      "grad_norm": 0.216796875,
      "learning_rate": 1.3666120056587068e-07,
      "loss": 0.1057,
      "num_tokens": 394332398.0,
      "reward": 0.4348958358168602,
      "reward_std": 0.23832784965634346,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 2300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.4583740234375,
      "epoch": 0.8837253960633702,
      "grad_norm": 0.251953125,
      "learning_rate": 1.3642292498176073e-07,
      "loss": 0.071,
      "num_tokens": 394494266.0,
      "reward": 0.6276041716337204,
      "reward_std": 0.36434168368577957,
      "rewards/accuracy_reward": 0.16666666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 2301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1359.5625457763672,
      "epoch": 0.8841094575132021,
      "grad_norm": 0.2421875,
      "learning_rate": 1.3618539358512548e-07,
      "loss": 0.0958,
      "num_tokens": 394638864.0,
      "reward": 0.6145833432674408,
      "reward_std": 0.23959755152463913,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4791666641831398,
      "step": 2302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.6562805175781,
      "epoch": 0.8844935189630341,
      "grad_norm": 0.23828125,
      "learning_rate": 1.3594860680337713e-07,
      "loss": 0.1635,
      "num_tokens": 394803751.0,
      "reward": 0.518229179084301,
      "reward_std": 0.2534041032195091,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 2303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.3229675292969,
      "epoch": 0.884877580412866,
      "grad_norm": 0.205078125,
      "learning_rate": 1.3571256506258796e-07,
      "loss": 0.135,
      "num_tokens": 394986726.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.25103478133678436,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 2304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1554.9688110351562,
      "epoch": 0.8852616418626981,
      "grad_norm": 0.2392578125,
      "learning_rate": 1.3547726878748966e-07,
      "loss": 0.0754,
      "num_tokens": 395149899.0,
      "reward": 0.557291679084301,
      "reward_std": 0.24915126711130142,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 2305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.6875305175781,
      "epoch": 0.88564570331253,
      "grad_norm": 0.2060546875,
      "learning_rate": 1.3524271840147284e-07,
      "loss": 0.0795,
      "num_tokens": 395319653.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.1983170360326767,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 2306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.0000610351562,
      "epoch": 0.886029764762362,
      "grad_norm": 0.2275390625,
      "learning_rate": 1.3500891432658533e-07,
      "loss": 0.0862,
      "num_tokens": 395476117.0,
      "reward": 0.5494791865348816,
      "reward_std": 0.2777433916926384,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291567325592,
      "step": 2307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1463.6771240234375,
      "epoch": 0.886413826212194,
      "grad_norm": 0.236328125,
      "learning_rate": 1.347758569835326e-07,
      "loss": 0.0728,
      "num_tokens": 395629014.0,
      "reward": 0.5234375223517418,
      "reward_std": 0.19434730522334576,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 2308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1584.1458740234375,
      "epoch": 0.8867978876620259,
      "grad_norm": 0.21875,
      "learning_rate": 1.345435467916763e-07,
      "loss": 0.104,
      "num_tokens": 395796468.0,
      "reward": 0.494791679084301,
      "reward_std": 0.19822203367948532,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583283662796,
      "step": 2309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.9271545410156,
      "epoch": 0.8871819491118579,
      "grad_norm": 0.2294921875,
      "learning_rate": 1.3431198416903367e-07,
      "loss": 0.1178,
      "num_tokens": 395976133.0,
      "reward": 0.3645833432674408,
      "reward_std": 0.08881847001612186,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833358168602,
      "step": 2310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1456.0521240234375,
      "epoch": 0.8875660105616898,
      "grad_norm": 0.74609375,
      "learning_rate": 1.3408116953227672e-07,
      "loss": 0.0094,
      "num_tokens": 396127930.0,
      "reward": 0.549479179084301,
      "reward_std": 0.2412261851131916,
      "rewards/accuracy_reward": 0.12499999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 2311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.4479370117188,
      "epoch": 0.8879500720115219,
      "grad_norm": 0.2001953125,
      "learning_rate": 1.3385110329673142e-07,
      "loss": 0.1411,
      "num_tokens": 396295357.0,
      "reward": 0.5104166865348816,
      "reward_std": 0.22298242803663015,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 2312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.7916870117188,
      "epoch": 0.8883341334613538,
      "grad_norm": 0.19140625,
      "learning_rate": 1.3362178587637747e-07,
      "loss": 0.0914,
      "num_tokens": 396472097.0,
      "reward": 0.463541679084301,
      "reward_std": 0.18666276149451733,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 2313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1506.4271545410156,
      "epoch": 0.8887181949111858,
      "grad_norm": 0.2392578125,
      "learning_rate": 1.3339321768384673e-07,
      "loss": 0.0661,
      "num_tokens": 396630242.0,
      "reward": 0.536458358168602,
      "reward_std": 0.19612153992056847,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 2314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1682.8229675292969,
      "epoch": 0.8891022563610178,
      "grad_norm": 0.240234375,
      "learning_rate": 1.3316539913042315e-07,
      "loss": 0.0739,
      "num_tokens": 396805873.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.2666408605873585,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 2315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1737.0521240234375,
      "epoch": 0.8894863178108497,
      "grad_norm": 0.1943359375,
      "learning_rate": 1.3293833062604166e-07,
      "loss": 0.1081,
      "num_tokens": 396986454.0,
      "reward": 0.3671875149011612,
      "reward_std": 0.11585310753434896,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 2316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1692.6146240234375,
      "epoch": 0.8898703792606817,
      "grad_norm": 0.2333984375,
      "learning_rate": 1.3271201257928773e-07,
      "loss": 0.0819,
      "num_tokens": 397166065.0,
      "reward": 0.4505208507180214,
      "reward_std": 0.20136183500289917,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.5104370117188,
      "epoch": 0.8902544407105136,
      "grad_norm": 0.2275390625,
      "learning_rate": 1.3248644539739632e-07,
      "loss": 0.1367,
      "num_tokens": 397324682.0,
      "reward": 0.5390625149011612,
      "reward_std": 0.2547386419028044,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 2318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1624.1771240234375,
      "epoch": 0.8906385021603457,
      "grad_norm": 0.236328125,
      "learning_rate": 1.3226162948625126e-07,
      "loss": 0.1544,
      "num_tokens": 397495131.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.1916624940931797,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828124925494194,
      "step": 2319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.6771240234375,
      "epoch": 0.8910225636101776,
      "grad_norm": 0.23828125,
      "learning_rate": 1.3203756525038457e-07,
      "loss": 0.134,
      "num_tokens": 397658684.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.21710697747766972,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 2320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.7708740234375,
      "epoch": 0.8914066250600096,
      "grad_norm": 0.2314453125,
      "learning_rate": 1.3181425309297585e-07,
      "loss": 0.0852,
      "num_tokens": 397829918.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.2494318801909685,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 2321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1528.6354370117188,
      "epoch": 0.8917906865098416,
      "grad_norm": 0.2197265625,
      "learning_rate": 1.315916934158513e-07,
      "loss": 0.0505,
      "num_tokens": 397992683.0,
      "reward": 0.4322916865348816,
      "reward_std": 0.131908955052495,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 2322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1652.4063110351562,
      "epoch": 0.8921747479596736,
      "grad_norm": 0.2060546875,
      "learning_rate": 1.3136988661948305e-07,
      "loss": 0.0946,
      "num_tokens": 398166114.0,
      "reward": 0.4661458358168602,
      "reward_std": 0.1962353978306055,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 2323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1811.2187805175781,
      "epoch": 0.8925588094095055,
      "grad_norm": 0.2060546875,
      "learning_rate": 1.3114883310298865e-07,
      "loss": 0.0812,
      "num_tokens": 398357287.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.29946768656373024,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 2324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1675.4687805175781,
      "epoch": 0.8929428708593375,
      "grad_norm": 0.208984375,
      "learning_rate": 1.3092853326413017e-07,
      "loss": 0.0978,
      "num_tokens": 398530724.0,
      "reward": 0.5885416865348816,
      "reward_std": 0.2806946001946926,
      "rewards/accuracy_reward": 0.1875,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 2325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.9375610351562,
      "epoch": 0.8933269323091695,
      "grad_norm": 0.220703125,
      "learning_rate": 1.3070898749931344e-07,
      "loss": 0.1634,
      "num_tokens": 398696182.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.2623567171394825,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 2326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1577.2604675292969,
      "epoch": 0.8937109937590014,
      "grad_norm": 0.23046875,
      "learning_rate": 1.3049019620358733e-07,
      "loss": 0.1083,
      "num_tokens": 398860631.0,
      "reward": 0.5625000149011612,
      "reward_std": 0.32419707253575325,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 2327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.5417175292969,
      "epoch": 0.8940950552088334,
      "grad_norm": 0.2314453125,
      "learning_rate": 1.3027215977064348e-07,
      "loss": 0.0629,
      "num_tokens": 399019747.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.18802233040332794,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 2328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1416.4479370117188,
      "epoch": 0.8944791166586654,
      "grad_norm": 0.259765625,
      "learning_rate": 1.3005487859281487e-07,
      "loss": 0.0977,
      "num_tokens": 399170126.0,
      "reward": 0.5937500074505806,
      "reward_std": 0.32317201793193817,
      "rewards/accuracy_reward": 0.11458333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4791666716337204,
      "step": 2329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.0000610351562,
      "epoch": 0.8948631781084974,
      "grad_norm": 0.2138671875,
      "learning_rate": 1.298383530610756e-07,
      "loss": 0.0938,
      "num_tokens": 399343478.0,
      "reward": 0.5052083432674408,
      "reward_std": 0.22402063012123108,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906250074505806,
      "step": 2330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1628.3333740234375,
      "epoch": 0.8952472395583293,
      "grad_norm": 0.1767578125,
      "learning_rate": 1.2962258356504005e-07,
      "loss": 0.1174,
      "num_tokens": 399517758.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.123464479111135,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 2331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.9792022705078,
      "epoch": 0.8956313010081614,
      "grad_norm": 0.2734375,
      "learning_rate": 1.2940757049296242e-07,
      "loss": 0.1699,
      "num_tokens": 399673668.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.24557797983288765,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 2332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1268.1771240234375,
      "epoch": 0.8960153624579933,
      "grad_norm": 0.29296875,
      "learning_rate": 1.2919331423173544e-07,
      "loss": 0.1168,
      "num_tokens": 399808925.0,
      "reward": 0.5494791865348816,
      "reward_std": 0.2360467091202736,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5182291567325592,
      "step": 2333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.0208740234375,
      "epoch": 0.8963994239078252,
      "grad_norm": 0.1494140625,
      "learning_rate": 1.289798151668903e-07,
      "loss": 0.0861,
      "num_tokens": 399994007.0,
      "reward": 0.408854179084301,
      "reward_std": 0.1540621519088745,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 2334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1582.3750305175781,
      "epoch": 0.8967834853576572,
      "grad_norm": 0.2109375,
      "learning_rate": 1.2876707368259543e-07,
      "loss": 0.1156,
      "num_tokens": 400160355.0,
      "reward": 0.6119792014360428,
      "reward_std": 0.37378303706645966,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 2335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1480.1875610351562,
      "epoch": 0.8971675468074892,
      "grad_norm": 0.267578125,
      "learning_rate": 1.285550901616565e-07,
      "loss": 0.1611,
      "num_tokens": 400316453.0,
      "reward": 0.5859375298023224,
      "reward_std": 0.30789605900645256,
      "rewards/accuracy_reward": 0.12500000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 2336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1729.5521240234375,
      "epoch": 0.8975516082573212,
      "grad_norm": 0.181640625,
      "learning_rate": 1.28343864985515e-07,
      "loss": 0.1063,
      "num_tokens": 400496954.0,
      "reward": 0.440104179084301,
      "reward_std": 0.17960853315889835,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 2337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1427.3437805175781,
      "epoch": 0.8979356697071531,
      "grad_norm": 0.26171875,
      "learning_rate": 1.2813339853424776e-07,
      "loss": 0.0945,
      "num_tokens": 400647291.0,
      "reward": 0.5598958507180214,
      "reward_std": 0.21924669574946165,
      "rewards/accuracy_reward": 0.08333333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625074505806,
      "step": 2338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.5416870117188,
      "epoch": 0.8983197311569852,
      "grad_norm": 0.1962890625,
      "learning_rate": 1.279236911865667e-07,
      "loss": 0.1001,
      "num_tokens": 400810879.0,
      "reward": 0.6145833358168602,
      "reward_std": 0.29833519738167524,
      "rewards/accuracy_reward": 0.1770833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 2339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1629.7292175292969,
      "epoch": 0.8987037926068171,
      "grad_norm": 0.24609375,
      "learning_rate": 1.2771474331981761e-07,
      "loss": 0.057,
      "num_tokens": 400982845.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.1572512537240982,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062500074505806,
      "step": 2340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.3854675292969,
      "epoch": 0.899087854056649,
      "grad_norm": 0.189453125,
      "learning_rate": 1.275065553099797e-07,
      "loss": 0.0724,
      "num_tokens": 401173042.0,
      "reward": 0.4114583432674408,
      "reward_std": 0.15022391732782125,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 2341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.9896240234375,
      "epoch": 0.8994719155064811,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.2729912753166478e-07,
      "loss": 0.089,
      "num_tokens": 401361769.0,
      "reward": 0.4713541865348816,
      "reward_std": 0.1704718554392457,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 2342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1506.6875305175781,
      "epoch": 0.899855976956313,
      "grad_norm": 0.2294921875,
      "learning_rate": 1.27092460358117e-07,
      "loss": 0.1036,
      "num_tokens": 401519523.0,
      "reward": 0.5364583507180214,
      "reward_std": 0.28269535303115845,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 2343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1676.4791870117188,
      "epoch": 0.900240038406145,
      "grad_norm": 0.1943359375,
      "learning_rate": 1.268865541612117e-07,
      "loss": 0.078,
      "num_tokens": 401695177.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.18259229138493538,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1807.0208740234375,
      "epoch": 0.9006240998559769,
      "grad_norm": 0.1748046875,
      "learning_rate": 1.266814093114549e-07,
      "loss": 0.122,
      "num_tokens": 401882739.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.1761203990317881,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3359375,
      "step": 2345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1373.5521087646484,
      "epoch": 0.901008161305809,
      "grad_norm": 0.263671875,
      "learning_rate": 1.264770261779826e-07,
      "loss": 0.1501,
      "num_tokens": 402031400.0,
      "reward": 0.5390625298023224,
      "reward_std": 0.26147207617759705,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 2346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1836.3646240234375,
      "epoch": 0.9013922227556409,
      "grad_norm": 0.1884765625,
      "learning_rate": 1.2627340512856049e-07,
      "loss": 0.1108,
      "num_tokens": 402223523.0,
      "reward": 0.377604179084301,
      "reward_std": 0.1669241669587791,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3463541716337204,
      "step": 2347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.6875610351562,
      "epoch": 0.9017762842054728,
      "grad_norm": 0.1748046875,
      "learning_rate": 1.2607054652958264e-07,
      "loss": 0.0367,
      "num_tokens": 402409357.0,
      "reward": 0.4479166865348816,
      "reward_std": 0.16698754532262683,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3749999925494194,
      "step": 2348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1700.6875610351562,
      "epoch": 0.9021603456553049,
      "grad_norm": 0.21484375,
      "learning_rate": 1.2586845074607122e-07,
      "loss": 0.0973,
      "num_tokens": 402585631.0,
      "reward": 0.4843750223517418,
      "reward_std": 0.22015490755438805,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 2349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1356.1354522705078,
      "epoch": 0.9025444071051368,
      "grad_norm": 0.2470703125,
      "learning_rate": 1.2566711814167588e-07,
      "loss": 0.154,
      "num_tokens": 402728876.0,
      "reward": 0.5104166865348816,
      "reward_std": 0.20853115245699883,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5000000074505806,
      "step": 2350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1633.6146240234375,
      "epoch": 0.9029284685549688,
      "grad_norm": 0.2099609375,
      "learning_rate": 1.254665490786731e-07,
      "loss": 0.0775,
      "num_tokens": 402898375.0,
      "reward": 0.5052083507180214,
      "reward_std": 0.2241690456867218,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 2351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1819.7812805175781,
      "epoch": 0.9033125300048007,
      "grad_norm": 0.1806640625,
      "learning_rate": 1.2526674391796529e-07,
      "loss": 0.0937,
      "num_tokens": 403085362.0,
      "reward": 0.4036458358168602,
      "reward_std": 0.18787747249007225,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 2352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.0937805175781,
      "epoch": 0.9036965914546328,
      "grad_norm": 0.2265625,
      "learning_rate": 1.250677030190802e-07,
      "loss": 0.1168,
      "num_tokens": 403260611.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.29493550956249237,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958432674408,
      "step": 2353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1404.447982788086,
      "epoch": 0.9040806529044647,
      "grad_norm": 0.251953125,
      "learning_rate": 1.248694267401708e-07,
      "loss": 0.0835,
      "num_tokens": 403409398.0,
      "reward": 0.6093750074505806,
      "reward_std": 0.27266680262982845,
      "rewards/accuracy_reward": 0.15624999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250074505806,
      "step": 2354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1506.6979675292969,
      "epoch": 0.9044647143542967,
      "grad_norm": 0.23828125,
      "learning_rate": 1.2467191543801376e-07,
      "loss": 0.0797,
      "num_tokens": 403569449.0,
      "reward": 0.5364583432674408,
      "reward_std": 0.253981277346611,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 2355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1669.6458740234375,
      "epoch": 0.9048487758041287,
      "grad_norm": 0.1552734375,
      "learning_rate": 1.2447516946800949e-07,
      "loss": 0.0506,
      "num_tokens": 403742767.0,
      "reward": 0.5963541865348816,
      "reward_std": 0.278325661085546,
      "rewards/accuracy_reward": 0.1770833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 2356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1555.9479675292969,
      "epoch": 0.9052328372539606,
      "grad_norm": 0.25390625,
      "learning_rate": 1.2427918918418112e-07,
      "loss": 0.0932,
      "num_tokens": 403906802.0,
      "reward": 0.544270858168602,
      "reward_std": 0.19451238214969635,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375074505806,
      "step": 2357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1699.9687805175781,
      "epoch": 0.9056168987037926,
      "grad_norm": 0.2138671875,
      "learning_rate": 1.2408397493917414e-07,
      "loss": 0.0885,
      "num_tokens": 404086551.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.23142056539654732,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 2358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1453.3125305175781,
      "epoch": 0.9060009601536246,
      "grad_norm": 0.298828125,
      "learning_rate": 1.2388952708425568e-07,
      "loss": 0.0926,
      "num_tokens": 404242693.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.18196783028542995,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.450520820915699,
      "step": 2359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1326.791748046875,
      "epoch": 0.9063850216034566,
      "grad_norm": 0.216796875,
      "learning_rate": 1.2369584596931366e-07,
      "loss": 0.0569,
      "num_tokens": 404382177.0,
      "reward": 0.5546875223517418,
      "reward_std": 0.15175812505185604,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4921874925494194,
      "step": 2360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.4792175292969,
      "epoch": 0.9067690830532885,
      "grad_norm": 0.2294921875,
      "learning_rate": 1.235029319428564e-07,
      "loss": 0.1472,
      "num_tokens": 404551703.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.30004822090268135,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 2361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1585.3021240234375,
      "epoch": 0.9071531445031205,
      "grad_norm": 0.2197265625,
      "learning_rate": 1.2331078535201202e-07,
      "loss": 0.2035,
      "num_tokens": 404716964.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.22018585540354252,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875074505806,
      "step": 2362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1841.0833435058594,
      "epoch": 0.9075372059529525,
      "grad_norm": 0.17578125,
      "learning_rate": 1.2311940654252762e-07,
      "loss": 0.0755,
      "num_tokens": 404906460.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.20120700635015965,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 2363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1469.7083740234375,
      "epoch": 0.9079212674027844,
      "grad_norm": 0.26171875,
      "learning_rate": 1.2292879585876877e-07,
      "loss": 0.1342,
      "num_tokens": 405061112.0,
      "reward": 0.510416679084301,
      "reward_std": 0.2529658414423466,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 2364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1696.0625305175781,
      "epoch": 0.9083053288526164,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.2273895364371882e-07,
      "loss": 0.0951,
      "num_tokens": 405237718.0,
      "reward": 0.479166679084301,
      "reward_std": 0.19174599833786488,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 2365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1527.7812805175781,
      "epoch": 0.9086893903024484,
      "grad_norm": 0.22265625,
      "learning_rate": 1.225498802389786e-07,
      "loss": 0.0915,
      "num_tokens": 405397505.0,
      "reward": 0.5312500149011612,
      "reward_std": 0.2414370086044073,
      "rewards/accuracy_reward": 0.109375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 2366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1545.2292022705078,
      "epoch": 0.9090734517522804,
      "grad_norm": 0.58203125,
      "learning_rate": 1.2236157598476528e-07,
      "loss": 0.0119,
      "num_tokens": 405557071.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.18140976503491402,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.390625,
      "step": 2367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1589.4583740234375,
      "epoch": 0.9094575132021123,
      "grad_norm": 0.2001953125,
      "learning_rate": 1.2217404121991208e-07,
      "loss": 0.0602,
      "num_tokens": 405722643.0,
      "reward": 0.6250000298023224,
      "reward_std": 0.265402153134346,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 2368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.6979370117188,
      "epoch": 0.9098415746519443,
      "grad_norm": 0.197265625,
      "learning_rate": 1.2198727628186775e-07,
      "loss": 0.0758,
      "num_tokens": 405915702.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.157808193936944,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 2369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1853.1458740234375,
      "epoch": 0.9102256361017763,
      "grad_norm": 0.171875,
      "learning_rate": 1.218012815066956e-07,
      "loss": 0.1076,
      "num_tokens": 406108156.0,
      "reward": 0.385416679084301,
      "reward_std": 0.180924192070961,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.34375,
      "step": 2370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1472.6146240234375,
      "epoch": 0.9106096975516083,
      "grad_norm": 0.259765625,
      "learning_rate": 1.216160572290733e-07,
      "loss": 0.0753,
      "num_tokens": 406263847.0,
      "reward": 0.463541679084301,
      "reward_std": 0.16014512442052364,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531249925494194,
      "step": 2371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1318.6354370117188,
      "epoch": 0.9109937590014402,
      "grad_norm": 0.296875,
      "learning_rate": 1.2143160378229193e-07,
      "loss": 0.1246,
      "num_tokens": 406406068.0,
      "reward": 0.6328125149011612,
      "reward_std": 0.3000098317861557,
      "rewards/accuracy_reward": 0.14583333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4869791716337204,
      "step": 2372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1792.7604675292969,
      "epoch": 0.9113778204512722,
      "grad_norm": 0.2080078125,
      "learning_rate": 1.2124792149825576e-07,
      "loss": 0.0962,
      "num_tokens": 406592581.0,
      "reward": 0.40625,
      "reward_std": 0.17378820851445198,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 2373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.6563110351562,
      "epoch": 0.9117618819011042,
      "grad_norm": 0.181640625,
      "learning_rate": 1.2106501070748121e-07,
      "loss": 0.0767,
      "num_tokens": 406771196.0,
      "reward": 0.424479179084301,
      "reward_std": 0.14130015671253204,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 2374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.4583740234375,
      "epoch": 0.9121459433509361,
      "grad_norm": 0.2001953125,
      "learning_rate": 1.2088287173909667e-07,
      "loss": 0.1288,
      "num_tokens": 406945528.0,
      "reward": 0.4244791716337204,
      "reward_std": 0.16752923605963588,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1663.0312805175781,
      "epoch": 0.9125300048007681,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.207015049208415e-07,
      "loss": 0.1003,
      "num_tokens": 407120035.0,
      "reward": 0.5234375074505806,
      "reward_std": 0.2208158653229475,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 2376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.2083435058594,
      "epoch": 0.9129140662506001,
      "grad_norm": 0.263671875,
      "learning_rate": 1.2052091057906594e-07,
      "loss": 0.0972,
      "num_tokens": 407279103.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.20679353922605515,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 2377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1645.8646240234375,
      "epoch": 0.9132981277004321,
      "grad_norm": 0.2138671875,
      "learning_rate": 1.2034108903872996e-07,
      "loss": 0.1528,
      "num_tokens": 407452522.0,
      "reward": 0.408854179084301,
      "reward_std": 0.15106721315532923,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1463.0625305175781,
      "epoch": 0.913682189150264,
      "grad_norm": 0.263671875,
      "learning_rate": 1.2016204062340317e-07,
      "loss": 0.1376,
      "num_tokens": 407606664.0,
      "reward": 0.625,
      "reward_std": 0.29941441491246223,
      "rewards/accuracy_reward": 0.17708333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 2379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1640.7187805175781,
      "epoch": 0.914066250600096,
      "grad_norm": 0.1953125,
      "learning_rate": 1.1998376565526386e-07,
      "loss": 0.1073,
      "num_tokens": 407780085.0,
      "reward": 0.4635416865348816,
      "reward_std": 0.20584215596318245,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 2380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1732.5312805175781,
      "epoch": 0.914450312049928,
      "grad_norm": 0.208984375,
      "learning_rate": 1.1980626445509872e-07,
      "loss": 0.0368,
      "num_tokens": 407961456.0,
      "reward": 0.4427083432674408,
      "reward_std": 0.14900904521346092,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916567325592,
      "step": 2381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1595.6042175292969,
      "epoch": 0.9148343734997599,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.1962953734230212e-07,
      "loss": 0.0739,
      "num_tokens": 408129738.0,
      "reward": 0.4557291865348816,
      "reward_std": 0.18137486279010773,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958283662796,
      "step": 2382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1747.5000305175781,
      "epoch": 0.915218434949592,
      "grad_norm": 0.19921875,
      "learning_rate": 1.1945358463487537e-07,
      "loss": 0.1081,
      "num_tokens": 408310434.0,
      "reward": 0.4244791865348816,
      "reward_std": 0.19342019967734814,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515624925494194,
      "step": 2383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.9375610351562,
      "epoch": 0.9156024963994239,
      "grad_norm": 0.2119140625,
      "learning_rate": 1.1927840664942652e-07,
      "loss": 0.0985,
      "num_tokens": 408495820.0,
      "reward": 0.4505208507180214,
      "reward_std": 0.17474376782774925,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.9791870117188,
      "epoch": 0.9159865578492559,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.1910400370116958e-07,
      "loss": 0.1028,
      "num_tokens": 408664794.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.20526160253211856,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 2385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1511.1250305175781,
      "epoch": 0.9163706192990878,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.1893037610392382e-07,
      "loss": 0.0866,
      "num_tokens": 408824126.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.15404780209064484,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 2386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1488.8542175292969,
      "epoch": 0.9167546807489199,
      "grad_norm": 0.263671875,
      "learning_rate": 1.1875752417011339e-07,
      "loss": 0.1219,
      "num_tokens": 408980368.0,
      "reward": 0.4583333507180214,
      "reward_std": 0.16281879507005215,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.3437805175781,
      "epoch": 0.9171387421987518,
      "grad_norm": 0.2109375,
      "learning_rate": 1.1858544821076685e-07,
      "loss": 0.0792,
      "num_tokens": 409149193.0,
      "reward": 0.4531250298023224,
      "reward_std": 0.17100587859749794,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 2388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.1666870117188,
      "epoch": 0.9175228036485837,
      "grad_norm": 0.2373046875,
      "learning_rate": 1.1841414853551641e-07,
      "loss": 0.0891,
      "num_tokens": 409307401.0,
      "reward": 0.5833333656191826,
      "reward_std": 0.27352094277739525,
      "rewards/accuracy_reward": 0.125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 2389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1418.6146240234375,
      "epoch": 0.9179068650984158,
      "grad_norm": 0.275390625,
      "learning_rate": 1.1824362545259732e-07,
      "loss": 0.1025,
      "num_tokens": 409457276.0,
      "reward": 0.5338541716337204,
      "reward_std": 0.27937422692775726,
      "rewards/accuracy_reward": 0.07291666883975267,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375074505806,
      "step": 2390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1136.6666870117188,
      "epoch": 0.9182909265482477,
      "grad_norm": 0.275390625,
      "learning_rate": 1.1807387926884753e-07,
      "loss": 0.1574,
      "num_tokens": 409579660.0,
      "reward": 0.7552083432674408,
      "reward_std": 0.3045167252421379,
      "rewards/accuracy_reward": 0.2500000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5052083358168602,
      "step": 2391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1529.8646240234375,
      "epoch": 0.9186749879980797,
      "grad_norm": 0.251953125,
      "learning_rate": 1.179049102897071e-07,
      "loss": 0.1515,
      "num_tokens": 409739967.0,
      "reward": 0.5468750149011612,
      "reward_std": 0.31424589455127716,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 2392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1593.1145935058594,
      "epoch": 0.9190590494479116,
      "grad_norm": 0.251953125,
      "learning_rate": 1.1773671881921758e-07,
      "loss": 0.1463,
      "num_tokens": 409907730.0,
      "reward": 0.515625,
      "reward_std": 0.24578950926661491,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 2393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.9583740234375,
      "epoch": 0.9194431108977437,
      "grad_norm": 0.1875,
      "learning_rate": 1.1756930516002134e-07,
      "loss": 0.0576,
      "num_tokens": 410076174.0,
      "reward": 0.5130208432674408,
      "reward_std": 0.24346603732556105,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 2394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1443.6042175292969,
      "epoch": 0.9198271723475756,
      "grad_norm": 0.220703125,
      "learning_rate": 1.1740266961336127e-07,
      "loss": 0.1119,
      "num_tokens": 410228384.0,
      "reward": 0.6432291865348816,
      "reward_std": 0.2485102377831936,
      "rewards/accuracy_reward": 0.17708333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458283662796,
      "step": 2395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1862.1979675292969,
      "epoch": 0.9202112337974075,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.1723681247908007e-07,
      "loss": 0.0985,
      "num_tokens": 410421019.0,
      "reward": 0.3802083507180214,
      "reward_std": 0.18088404648005962,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3385416641831398,
      "step": 2396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.8750610351562,
      "epoch": 0.9205952952472396,
      "grad_norm": 0.193359375,
      "learning_rate": 1.1707173405562007e-07,
      "loss": 0.0588,
      "num_tokens": 410581519.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.16775843035429716,
      "rewards/accuracy_reward": 0.13541666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1598.7396240234375,
      "epoch": 0.9209793566970715,
      "grad_norm": 0.208984375,
      "learning_rate": 1.1690743464002187e-07,
      "loss": 0.0534,
      "num_tokens": 410747158.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.2530820332467556,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708283662796,
      "step": 2398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1745.1146240234375,
      "epoch": 0.9213634181469035,
      "grad_norm": 0.22265625,
      "learning_rate": 1.1674391452792471e-07,
      "loss": 0.1593,
      "num_tokens": 410929433.0,
      "reward": 0.3906250074505806,
      "reward_std": 0.199986070394516,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 2399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1602.6458435058594,
      "epoch": 0.9217474795967355,
      "grad_norm": 0.2265625,
      "learning_rate": 1.1658117401356547e-07,
      "loss": 0.1424,
      "num_tokens": 411097655.0,
      "reward": 0.4687500074505806,
      "reward_std": 0.2165900133550167,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1833.2916870117188,
      "epoch": 0.9221315410465675,
      "grad_norm": 0.19921875,
      "learning_rate": 1.1641921338977814e-07,
      "loss": 0.0791,
      "num_tokens": 411290147.0,
      "reward": 0.4036458507180214,
      "reward_std": 0.1726490268483758,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 2401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.6667175292969,
      "epoch": 0.9225156024963994,
      "grad_norm": 0.216796875,
      "learning_rate": 1.1625803294799343e-07,
      "loss": 0.1024,
      "num_tokens": 411454651.0,
      "reward": 0.604166679084301,
      "reward_std": 0.2286203783005476,
      "rewards/accuracy_reward": 0.15625000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 2402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.2500610351562,
      "epoch": 0.9228996639462314,
      "grad_norm": 0.25,
      "learning_rate": 1.1609763297823831e-07,
      "loss": 0.1192,
      "num_tokens": 411614011.0,
      "reward": 0.4609375223517418,
      "reward_std": 0.19187624473124743,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296874925494194,
      "step": 2403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1755.5313110351562,
      "epoch": 0.9232837253960634,
      "grad_norm": 0.2021484375,
      "learning_rate": 1.1593801376913523e-07,
      "loss": 0.1041,
      "num_tokens": 411798766.0,
      "reward": 0.497395858168602,
      "reward_std": 0.16202891431748867,
      "rewards/accuracy_reward": 0.1250000037252903,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 2404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.5313110351562,
      "epoch": 0.9236677868458953,
      "grad_norm": 0.216796875,
      "learning_rate": 1.1577917560790182e-07,
      "loss": 0.1052,
      "num_tokens": 411976249.0,
      "reward": 0.4739583358168602,
      "reward_std": 0.24472630769014359,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 2405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.5625305175781,
      "epoch": 0.9240518482957273,
      "grad_norm": 0.193359375,
      "learning_rate": 1.1562111878035018e-07,
      "loss": 0.0885,
      "num_tokens": 412155575.0,
      "reward": 0.5078125149011612,
      "reward_std": 0.24053557962179184,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 2406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.2917175292969,
      "epoch": 0.9244359097455593,
      "grad_norm": 0.23046875,
      "learning_rate": 1.1546384357088667e-07,
      "loss": 0.0774,
      "num_tokens": 412321139.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.21364685148000717,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 2407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.4479675292969,
      "epoch": 0.9248199711953913,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.153073502625111e-07,
      "loss": 0.1081,
      "num_tokens": 412488454.0,
      "reward": 0.5598958507180214,
      "reward_std": 0.17702364176511765,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 2408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1845.6145935058594,
      "epoch": 0.9252040326452232,
      "grad_norm": 0.1962890625,
      "learning_rate": 1.1515163913681636e-07,
      "loss": 0.0699,
      "num_tokens": 412681241.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.1991454679518938,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3645833432674408,
      "step": 2409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.4062805175781,
      "epoch": 0.9255880940950552,
      "grad_norm": 0.20703125,
      "learning_rate": 1.1499671047398777e-07,
      "loss": 0.1014,
      "num_tokens": 412848536.0,
      "reward": 0.557291679084301,
      "reward_std": 0.20216931775212288,
      "rewards/accuracy_reward": 0.1770833395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 2410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.9166870117188,
      "epoch": 0.9259721555448872,
      "grad_norm": 0.251953125,
      "learning_rate": 1.1484256455280286e-07,
      "loss": 0.0935,
      "num_tokens": 413010576.0,
      "reward": 0.4505208432674408,
      "reward_std": 0.16292519494891167,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 2411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1704.3645935058594,
      "epoch": 0.9263562169947192,
      "grad_norm": 0.208984375,
      "learning_rate": 1.146892016506308e-07,
      "loss": 0.1185,
      "num_tokens": 413188579.0,
      "reward": 0.5520833432674408,
      "reward_std": 0.24349429458379745,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 2412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.1042175292969,
      "epoch": 0.9267402784445511,
      "grad_norm": 0.3359375,
      "learning_rate": 1.1453662204343142e-07,
      "loss": 0.1121,
      "num_tokens": 413347789.0,
      "reward": 0.549479179084301,
      "reward_std": 0.24027040600776672,
      "rewards/accuracy_reward": 0.10416666511446238,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125074505806,
      "step": 2413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1414.7083435058594,
      "epoch": 0.9271243398943831,
      "grad_norm": 0.265625,
      "learning_rate": 1.1438482600575545e-07,
      "loss": 0.1283,
      "num_tokens": 413498041.0,
      "reward": 0.5859375149011612,
      "reward_std": 0.26063311845064163,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 2414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.0417175292969,
      "epoch": 0.9275084013442151,
      "grad_norm": 0.22265625,
      "learning_rate": 1.1423381381074342e-07,
      "loss": 0.0654,
      "num_tokens": 413675653.0,
      "reward": 0.424479179084301,
      "reward_std": 0.20266981795430183,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 2415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.1250305175781,
      "epoch": 0.927892462794047,
      "grad_norm": 0.2294921875,
      "learning_rate": 1.1408358573012569e-07,
      "loss": 0.1119,
      "num_tokens": 413840465.0,
      "reward": 0.5000000223517418,
      "reward_std": 0.17950892634689808,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.6354675292969,
      "epoch": 0.9282765242438791,
      "grad_norm": 0.2294921875,
      "learning_rate": 1.1393414203422138e-07,
      "loss": 0.0857,
      "num_tokens": 413999238.0,
      "reward": 0.6041666865348816,
      "reward_std": 0.2566812336444855,
      "rewards/accuracy_reward": 0.1406249962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4687500074505806,
      "step": 2417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.6875305175781,
      "epoch": 0.928660585693711,
      "grad_norm": 0.2294921875,
      "learning_rate": 1.1378548299193848e-07,
      "loss": 0.0845,
      "num_tokens": 414162464.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.19257643818855286,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1630.3854370117188,
      "epoch": 0.929044647143543,
      "grad_norm": 0.25,
      "learning_rate": 1.1363760887077281e-07,
      "loss": 0.1359,
      "num_tokens": 414335437.0,
      "reward": 0.4401041716337204,
      "reward_std": 0.18184654414653778,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 2419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1683.8854370117188,
      "epoch": 0.9294287085933749,
      "grad_norm": 0.1806640625,
      "learning_rate": 1.1349051993680815e-07,
      "loss": 0.0672,
      "num_tokens": 414510882.0,
      "reward": 0.4635416716337204,
      "reward_std": 0.19189088232815266,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 2420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1650.1146240234375,
      "epoch": 0.929812770043207,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.1334421645471499e-07,
      "loss": 0.0989,
      "num_tokens": 414681821.0,
      "reward": 0.4531250149011612,
      "reward_std": 0.20552089996635914,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 2421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1321.9166870117188,
      "epoch": 0.9301968314930389,
      "grad_norm": 0.255859375,
      "learning_rate": 1.131986986877509e-07,
      "loss": 0.0729,
      "num_tokens": 414824917.0,
      "reward": 0.653645858168602,
      "reward_std": 0.27512159012258053,
      "rewards/accuracy_reward": 0.2135416641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 2422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1764.5416870117188,
      "epoch": 0.9305808929428708,
      "grad_norm": 0.2001953125,
      "learning_rate": 1.1305396689775939e-07,
      "loss": 0.1185,
      "num_tokens": 415010553.0,
      "reward": 0.4192708432674408,
      "reward_std": 0.20091309025883675,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.0208740234375,
      "epoch": 0.9309649543927029,
      "grad_norm": 0.1865234375,
      "learning_rate": 1.1291002134516972e-07,
      "loss": 0.0918,
      "num_tokens": 415173371.0,
      "reward": 0.4713541716337204,
      "reward_std": 0.17490990832448006,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 2424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1512.916748046875,
      "epoch": 0.9313490158425348,
      "grad_norm": 0.2451171875,
      "learning_rate": 1.1276686228899645e-07,
      "loss": 0.0995,
      "num_tokens": 415332299.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.2538525015115738,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541716337204,
      "step": 2425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1645.6354370117188,
      "epoch": 0.9317330772923668,
      "grad_norm": 0.208984375,
      "learning_rate": 1.1262448998683888e-07,
      "loss": 0.129,
      "num_tokens": 415503608.0,
      "reward": 0.4505208507180214,
      "reward_std": 0.1865181289613247,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 2426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1500.0625305175781,
      "epoch": 0.9321171387421987,
      "grad_norm": 0.2275390625,
      "learning_rate": 1.124829046948808e-07,
      "loss": 0.1245,
      "num_tokens": 415660846.0,
      "reward": 0.572916679084301,
      "reward_std": 0.27939973026514053,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 2427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.2812805175781,
      "epoch": 0.9325012001920308,
      "grad_norm": 0.1787109375,
      "learning_rate": 1.1234210666788954e-07,
      "loss": 0.115,
      "num_tokens": 415838193.0,
      "reward": 0.4401041865348816,
      "reward_std": 0.1707091499119997,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 2428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1679.3542175292969,
      "epoch": 0.9328852616418627,
      "grad_norm": 0.193359375,
      "learning_rate": 1.122020961592161e-07,
      "loss": 0.1094,
      "num_tokens": 416014835.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.24807380512356758,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 2429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1591.7187805175781,
      "epoch": 0.9332693230916946,
      "grad_norm": 0.1953125,
      "learning_rate": 1.1206287342079427e-07,
      "loss": 0.0758,
      "num_tokens": 416181808.0,
      "reward": 0.5156250149011612,
      "reward_std": 0.21295772306621075,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 2430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1666.5521240234375,
      "epoch": 0.9336533845415267,
      "grad_norm": 0.189453125,
      "learning_rate": 1.119244387031406e-07,
      "loss": 0.0949,
      "num_tokens": 416357269.0,
      "reward": 0.4270833507180214,
      "reward_std": 0.18360981345176697,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 2431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1558.7917175292969,
      "epoch": 0.9340374459913586,
      "grad_norm": 0.21875,
      "learning_rate": 1.1178679225535315e-07,
      "loss": 0.1484,
      "num_tokens": 416520241.0,
      "reward": 0.5859375298023224,
      "reward_std": 0.30549411475658417,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875074505806,
      "step": 2432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1776.3229675292969,
      "epoch": 0.9344215074411906,
      "grad_norm": 0.18359375,
      "learning_rate": 1.1164993432511217e-07,
      "loss": 0.0815,
      "num_tokens": 416704576.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.14279170660302043,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 2433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1543.3333435058594,
      "epoch": 0.9348055688910225,
      "grad_norm": 0.2119140625,
      "learning_rate": 1.1151386515867864e-07,
      "loss": 0.1076,
      "num_tokens": 416864912.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.15195129811763763,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 2434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1449.3125610351562,
      "epoch": 0.9351896303408546,
      "grad_norm": 0.283203125,
      "learning_rate": 1.1137858500089453e-07,
      "loss": 0.0782,
      "num_tokens": 417017550.0,
      "reward": 0.645833358168602,
      "reward_std": 0.24015659652650356,
      "rewards/accuracy_reward": 0.1770833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 2435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.0417175292969,
      "epoch": 0.9355736917906865,
      "grad_norm": 0.2431640625,
      "learning_rate": 1.1124409409518176e-07,
      "loss": 0.114,
      "num_tokens": 417180234.0,
      "reward": 0.5208333507180214,
      "reward_std": 0.25980837270617485,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 2436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1574.0521545410156,
      "epoch": 0.9359577532405184,
      "grad_norm": 0.28515625,
      "learning_rate": 1.1111039268354243e-07,
      "loss": 0.107,
      "num_tokens": 417345215.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.2737583853304386,
      "rewards/accuracy_reward": 0.10416666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 2437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1586.8854675292969,
      "epoch": 0.9363418146903505,
      "grad_norm": 0.20703125,
      "learning_rate": 1.1097748100655772e-07,
      "loss": 0.1412,
      "num_tokens": 417511540.0,
      "reward": 0.4531250074505806,
      "reward_std": 0.15311330556869507,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 2438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1532.40625,
      "epoch": 0.9367258761401824,
      "grad_norm": 0.228515625,
      "learning_rate": 1.1084535930338805e-07,
      "loss": 0.1108,
      "num_tokens": 417674315.0,
      "reward": 0.5130208507180214,
      "reward_std": 0.19318412244319916,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375074505806,
      "step": 2439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1661.9584045410156,
      "epoch": 0.9371099375900144,
      "grad_norm": 0.22265625,
      "learning_rate": 1.1071402781177198e-07,
      "loss": 0.1414,
      "num_tokens": 417846903.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.2635497488081455,
      "rewards/accuracy_reward": 0.09375000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1536.9063110351562,
      "epoch": 0.9374939990398464,
      "grad_norm": 0.263671875,
      "learning_rate": 1.1058348676802658e-07,
      "loss": 0.0937,
      "num_tokens": 418008438.0,
      "reward": 0.5781250223517418,
      "reward_std": 0.25820716470479965,
      "rewards/accuracy_reward": 0.14583333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916716337204,
      "step": 2441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.2187805175781,
      "epoch": 0.9378780604896784,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.1045373640704647e-07,
      "loss": 0.0771,
      "num_tokens": 418188851.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.1917164847254753,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671874925494194,
      "step": 2442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1758.7395935058594,
      "epoch": 0.9382621219395103,
      "grad_norm": 0.1748046875,
      "learning_rate": 1.1032477696230333e-07,
      "loss": 0.0797,
      "num_tokens": 418371658.0,
      "reward": 0.408854179084301,
      "reward_std": 0.1754542849957943,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 2443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.0312805175781,
      "epoch": 0.9386461833893422,
      "grad_norm": 0.244140625,
      "learning_rate": 1.1019660866584593e-07,
      "loss": 0.1393,
      "num_tokens": 418541461.0,
      "reward": 0.6197916865348816,
      "reward_std": 0.3179861269891262,
      "rewards/accuracy_reward": 0.21874999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 2444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1784.5000305175781,
      "epoch": 0.9390302448391743,
      "grad_norm": 0.185546875,
      "learning_rate": 1.1006923174829917e-07,
      "loss": 0.0712,
      "num_tokens": 418726757.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.18856918439269066,
      "rewards/accuracy_reward": 0.1145833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 2445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1513.6146240234375,
      "epoch": 0.9394143062890062,
      "grad_norm": 0.240234375,
      "learning_rate": 1.0994264643886437e-07,
      "loss": 0.1429,
      "num_tokens": 418888928.0,
      "reward": 0.5703125149011612,
      "reward_std": 0.2852921672165394,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291716337204,
      "step": 2446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1413.375015258789,
      "epoch": 0.9397983677388382,
      "grad_norm": 0.24609375,
      "learning_rate": 1.0981685296531794e-07,
      "loss": 0.1208,
      "num_tokens": 419038132.0,
      "reward": 0.5781250223517418,
      "reward_std": 0.20207114331424236,
      "rewards/accuracy_reward": 0.1249999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.453125,
      "step": 2447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.0729675292969,
      "epoch": 0.9401824291886702,
      "grad_norm": 0.2099609375,
      "learning_rate": 1.096918515540119e-07,
      "loss": 0.0978,
      "num_tokens": 419218587.0,
      "reward": 0.416666679084301,
      "reward_std": 0.1748107634484768,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4062499925494194,
      "step": 2448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.0729675292969,
      "epoch": 0.9405664906385022,
      "grad_norm": 0.2080078125,
      "learning_rate": 1.0956764242987268e-07,
      "loss": 0.1077,
      "num_tokens": 419398730.0,
      "reward": 0.502604179084301,
      "reward_std": 0.26557010039687157,
      "rewards/accuracy_reward": 0.11458333116024733,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 2449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1668.7396240234375,
      "epoch": 0.9409505520883341,
      "grad_norm": 0.2021484375,
      "learning_rate": 1.094442258164014e-07,
      "loss": 0.1207,
      "num_tokens": 419572601.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.19827931001782417,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 2450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.6250305175781,
      "epoch": 0.941334613538166,
      "grad_norm": 0.228515625,
      "learning_rate": 1.0932160193567281e-07,
      "loss": 0.1323,
      "num_tokens": 419738741.0,
      "reward": 0.463541679084301,
      "reward_std": 0.18898312747478485,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 2451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1789.1667175292969,
      "epoch": 0.9417186749879981,
      "grad_norm": 0.1787109375,
      "learning_rate": 1.0919977100833553e-07,
      "loss": 0.1065,
      "num_tokens": 419925141.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.1555524282157421,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125074505806,
      "step": 2452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.9583435058594,
      "epoch": 0.94210273643783,
      "grad_norm": 0.255859375,
      "learning_rate": 1.090787332536112e-07,
      "loss": 0.0696,
      "num_tokens": 420091113.0,
      "reward": 0.5026041865348816,
      "reward_std": 0.25314757227897644,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 2453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1466.3854675292969,
      "epoch": 0.942486797887662,
      "grad_norm": 0.302734375,
      "learning_rate": 1.0895848888929424e-07,
      "loss": 0.1197,
      "num_tokens": 420245406.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.29645889066159725,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 2454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1491.0209045410156,
      "epoch": 0.942870859337494,
      "grad_norm": 0.283203125,
      "learning_rate": 1.0883903813175141e-07,
      "loss": 0.0559,
      "num_tokens": 420400992.0,
      "reward": 0.5364583432674408,
      "reward_std": 0.28926974162459373,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 2455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1530.2188110351562,
      "epoch": 0.943254920787326,
      "grad_norm": 0.275390625,
      "learning_rate": 1.087203811959215e-07,
      "loss": 0.1534,
      "num_tokens": 420564189.0,
      "reward": 0.4791666865348816,
      "reward_std": 0.18202018178999424,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1531.0104370117188,
      "epoch": 0.9436389822371579,
      "grad_norm": 0.2431640625,
      "learning_rate": 1.0860251829531497e-07,
      "loss": 0.0949,
      "num_tokens": 420728518.0,
      "reward": 0.552083358168602,
      "reward_std": 0.23649240098893642,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333283662796,
      "step": 2457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1507.8229675292969,
      "epoch": 0.94402304368699,
      "grad_norm": 0.2392578125,
      "learning_rate": 1.0848544964201341e-07,
      "loss": 0.1124,
      "num_tokens": 420884677.0,
      "reward": 0.5390625149011612,
      "reward_std": 0.24009912833571434,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765624925494194,
      "step": 2458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1717.1667175292969,
      "epoch": 0.9444071051368219,
      "grad_norm": 0.2470703125,
      "learning_rate": 1.0836917544666927e-07,
      "loss": 0.0119,
      "num_tokens": 421064413.0,
      "reward": 0.4088541716337204,
      "reward_std": 0.136031249538064,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041567325592,
      "step": 2459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1749.5312805175781,
      "epoch": 0.9447911665866539,
      "grad_norm": 0.17578125,
      "learning_rate": 1.0825369591850543e-07,
      "loss": 0.0856,
      "num_tokens": 421244120.0,
      "reward": 0.481770858168602,
      "reward_std": 0.2714114133268595,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 2460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.4583740234375,
      "epoch": 0.9451752280364858,
      "grad_norm": 0.212890625,
      "learning_rate": 1.0813901126531494e-07,
      "loss": 0.144,
      "num_tokens": 421412908.0,
      "reward": 0.401041679084301,
      "reward_std": 0.12174698244780302,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 2461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1525.4688110351562,
      "epoch": 0.9455592894863178,
      "grad_norm": 0.1748046875,
      "learning_rate": 1.0802512169346048e-07,
      "loss": 0.0837,
      "num_tokens": 421576633.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.221262127161026,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 2462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1723.0417175292969,
      "epoch": 0.9459433509361498,
      "grad_norm": 0.2255859375,
      "learning_rate": 1.0791202740787404e-07,
      "loss": 0.0664,
      "num_tokens": 421756701.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.18414651229977608,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 2463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.1875610351562,
      "epoch": 0.9463274123859817,
      "grad_norm": 0.2177734375,
      "learning_rate": 1.0779972861205668e-07,
      "loss": 0.1262,
      "num_tokens": 421926647.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.21767477504909039,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3906250074505806,
      "step": 2464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.2083435058594,
      "epoch": 0.9467114738358138,
      "grad_norm": 0.2119140625,
      "learning_rate": 1.0768822550807808e-07,
      "loss": 0.1102,
      "num_tokens": 422107179.0,
      "reward": 0.5807291939854622,
      "reward_std": 0.34345704317092896,
      "rewards/accuracy_reward": 0.17708333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458283662796,
      "step": 2465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.3854675292969,
      "epoch": 0.9470955352856457,
      "grad_norm": 0.212890625,
      "learning_rate": 1.0757751829657588e-07,
      "loss": 0.1182,
      "num_tokens": 422292696.0,
      "reward": 0.3984375149011612,
      "reward_std": 0.15911060757935047,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3567708358168602,
      "step": 2466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.9896545410156,
      "epoch": 0.9474795967354777,
      "grad_norm": 0.2021484375,
      "learning_rate": 1.0746760717675599e-07,
      "loss": 0.0559,
      "num_tokens": 422450775.0,
      "reward": 0.5416666939854622,
      "reward_std": 0.17631728388369083,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375,
      "step": 2467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1786.9271240234375,
      "epoch": 0.9478636581853096,
      "grad_norm": 0.18359375,
      "learning_rate": 1.0735849234639149e-07,
      "loss": 0.1086,
      "num_tokens": 422635168.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.21187441796064377,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3697916716337204,
      "step": 2468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1424.1146240234375,
      "epoch": 0.9482477196351417,
      "grad_norm": 0.294921875,
      "learning_rate": 1.0725017400182305e-07,
      "loss": 0.063,
      "num_tokens": 422786259.0,
      "reward": 0.5286458507180214,
      "reward_std": 0.2555670626461506,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4557291567325592,
      "step": 2469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1685.5729370117188,
      "epoch": 0.9486317810849736,
      "grad_norm": 0.25390625,
      "learning_rate": 1.0714265233795757e-07,
      "loss": 0.0571,
      "num_tokens": 422964930.0,
      "reward": 0.4062500149011612,
      "reward_std": 0.1560169793665409,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 2470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1600.0312805175781,
      "epoch": 0.9490158425348055,
      "grad_norm": 0.2060546875,
      "learning_rate": 1.0703592754826887e-07,
      "loss": 0.1013,
      "num_tokens": 423133277.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.2187245413661003,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 2471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1611.5104370117188,
      "epoch": 0.9493999039846376,
      "grad_norm": 0.2392578125,
      "learning_rate": 1.0692999982479679e-07,
      "loss": 0.1273,
      "num_tokens": 423303054.0,
      "reward": 0.5208333507180214,
      "reward_std": 0.23288190737366676,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 2472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.9479370117188,
      "epoch": 0.9497839654344695,
      "grad_norm": 0.2060546875,
      "learning_rate": 1.0682486935814677e-07,
      "loss": 0.0581,
      "num_tokens": 423472585.0,
      "reward": 0.4270833432674408,
      "reward_std": 0.14702552650123835,
      "rewards/accuracy_reward": 0.0520833320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3749999925494194,
      "step": 2473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.9375610351562,
      "epoch": 0.9501680268843015,
      "grad_norm": 0.1845703125,
      "learning_rate": 1.0672053633748988e-07,
      "loss": 0.1007,
      "num_tokens": 423649459.0,
      "reward": 0.4661458358168602,
      "reward_std": 0.2019666749984026,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958432674408,
      "step": 2474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1656.9375305175781,
      "epoch": 0.9505520883341335,
      "grad_norm": 0.1953125,
      "learning_rate": 1.0661700095056206e-07,
      "loss": 0.1471,
      "num_tokens": 423825397.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.15266925748437643,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041716337204,
      "step": 2475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1607.5104675292969,
      "epoch": 0.9509361497839655,
      "grad_norm": 0.21484375,
      "learning_rate": 1.0651426338366419e-07,
      "loss": 0.1064,
      "num_tokens": 423991966.0,
      "reward": 0.544270858168602,
      "reward_std": 0.3008243516087532,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 2476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1565.7604370117188,
      "epoch": 0.9513202112337974,
      "grad_norm": 0.2119140625,
      "learning_rate": 1.0641232382166143e-07,
      "loss": 0.1188,
      "num_tokens": 424158343.0,
      "reward": 0.463541679084301,
      "reward_std": 0.20522635243833065,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 2477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1416.2083435058594,
      "epoch": 0.9517042726836293,
      "grad_norm": 0.2138671875,
      "learning_rate": 1.0631118244798309e-07,
      "loss": 0.08,
      "num_tokens": 424308011.0,
      "reward": 0.6119791865348816,
      "reward_std": 0.3072926625609398,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625074505806,
      "step": 2478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1852.9063110351562,
      "epoch": 0.9520883341334614,
      "grad_norm": 0.16015625,
      "learning_rate": 1.0621083944462219e-07,
      "loss": 0.0891,
      "num_tokens": 424501578.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.16665074555203319,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3489583358168602,
      "step": 2479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1695.4687805175781,
      "epoch": 0.9524723955832933,
      "grad_norm": 0.208984375,
      "learning_rate": 1.0611129499213514e-07,
      "loss": 0.0994,
      "num_tokens": 424679471.0,
      "reward": 0.4817708432674408,
      "reward_std": 0.22870320826768875,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375074505806,
      "step": 2480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1490.8020935058594,
      "epoch": 0.9528564570331253,
      "grad_norm": 0.267578125,
      "learning_rate": 1.0601254926964153e-07,
      "loss": 0.0763,
      "num_tokens": 424836356.0,
      "reward": 0.6041666865348816,
      "reward_std": 0.2786046750843525,
      "rewards/accuracy_reward": 0.1770833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833432674408,
      "step": 2481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1748.4792175292969,
      "epoch": 0.9532405184829573,
      "grad_norm": 0.1962890625,
      "learning_rate": 1.059146024548237e-07,
      "loss": 0.1155,
      "num_tokens": 425016514.0,
      "reward": 0.3958333507180214,
      "reward_std": 0.15023773722350597,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 2482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1787.6771240234375,
      "epoch": 0.9536245799327893,
      "grad_norm": 0.1787109375,
      "learning_rate": 1.0581745472392626e-07,
      "loss": 0.0444,
      "num_tokens": 425203251.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.19783409871160984,
      "rewards/accuracy_reward": 0.1041666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958283662796,
      "step": 2483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1619.5417175292969,
      "epoch": 0.9540086413826212,
      "grad_norm": 0.1640625,
      "learning_rate": 1.0572110625175626e-07,
      "loss": 0.0561,
      "num_tokens": 425372623.0,
      "reward": 0.604166679084301,
      "reward_std": 0.26472804648801684,
      "rewards/accuracy_reward": 0.18750000651925802,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 2484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1719.8958740234375,
      "epoch": 0.9543927028324531,
      "grad_norm": 0.1904296875,
      "learning_rate": 1.0562555721168232e-07,
      "loss": 0.1414,
      "num_tokens": 425552029.0,
      "reward": 0.4713541865348816,
      "reward_std": 0.2119925208389759,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 2485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1583.7916870117188,
      "epoch": 0.9547767642822852,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.0553080777563474e-07,
      "loss": 0.1012,
      "num_tokens": 425720113.0,
      "reward": 0.6875000298023224,
      "reward_std": 0.3311767615377903,
      "rewards/accuracy_reward": 0.2604166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 2486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1730.7188110351562,
      "epoch": 0.9551608257321171,
      "grad_norm": 0.1806640625,
      "learning_rate": 1.0543685811410487e-07,
      "loss": 0.0665,
      "num_tokens": 425902806.0,
      "reward": 0.3880208432674408,
      "reward_std": 0.11093655228614807,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208283662796,
      "step": 2487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1445.9166870117188,
      "epoch": 0.9555448871819491,
      "grad_norm": 0.2578125,
      "learning_rate": 1.0534370839614507e-07,
      "loss": 0.0806,
      "num_tokens": 426056534.0,
      "reward": 0.4817708507180214,
      "reward_std": 0.23845256865024567,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041567325592,
      "step": 2488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1608.2812805175781,
      "epoch": 0.9559289486317811,
      "grad_norm": 0.220703125,
      "learning_rate": 1.0525135878936825e-07,
      "loss": 0.0864,
      "num_tokens": 426223769.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.1580763477832079,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 2489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.7292175292969,
      "epoch": 0.9563130100816131,
      "grad_norm": 0.2197265625,
      "learning_rate": 1.0515980945994754e-07,
      "loss": 0.0928,
      "num_tokens": 426385639.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.2934465706348419,
      "rewards/accuracy_reward": 0.1145833358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791716337204,
      "step": 2490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.8437805175781,
      "epoch": 0.956697071531445,
      "grad_norm": 0.2314453125,
      "learning_rate": 1.0506906057261617e-07,
      "loss": 0.16,
      "num_tokens": 426556944.0,
      "reward": 0.484375,
      "reward_std": 0.2563592791557312,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 2491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.7083435058594,
      "epoch": 0.957081132981277,
      "grad_norm": 0.2314453125,
      "learning_rate": 1.0497911229066702e-07,
      "loss": 0.0831,
      "num_tokens": 426734460.0,
      "reward": 0.5781250298023224,
      "reward_std": 0.20210767164826393,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 2492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1686.9479675292969,
      "epoch": 0.957465194431109,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.0488996477595223e-07,
      "loss": 0.058,
      "num_tokens": 426910911.0,
      "reward": 0.4739583358168602,
      "reward_std": 0.25122835859656334,
      "rewards/accuracy_reward": 0.09375000279396772,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 2493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1723.6459045410156,
      "epoch": 0.9578492558809409,
      "grad_norm": 0.177734375,
      "learning_rate": 1.0480161818888331e-07,
      "loss": 0.0805,
      "num_tokens": 427091693.0,
      "reward": 0.4583333432674408,
      "reward_std": 0.1858641430735588,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333283662796,
      "step": 2494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1479.6667175292969,
      "epoch": 0.9582333173307729,
      "grad_norm": 0.2734375,
      "learning_rate": 1.0471407268843039e-07,
      "loss": 0.0833,
      "num_tokens": 427248061.0,
      "reward": 0.4921875149011612,
      "reward_std": 0.21550990641117096,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041567325592,
      "step": 2495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.2187805175781,
      "epoch": 0.9586173787806049,
      "grad_norm": 0.197265625,
      "learning_rate": 1.0462732843212213e-07,
      "loss": 0.0881,
      "num_tokens": 427429690.0,
      "reward": 0.4583333358168602,
      "reward_std": 0.2508394196629524,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 2496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1451.4791870117188,
      "epoch": 0.9590014402304369,
      "grad_norm": 0.2373046875,
      "learning_rate": 1.0454138557604556e-07,
      "loss": 0.1014,
      "num_tokens": 427581424.0,
      "reward": 0.5703125149011612,
      "reward_std": 0.28095727041363716,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 2497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1270.9375610351562,
      "epoch": 0.9593855016802688,
      "grad_norm": 0.26171875,
      "learning_rate": 1.0445624427484561e-07,
      "loss": 0.1676,
      "num_tokens": 427716858.0,
      "reward": 0.5625000074505806,
      "reward_std": 0.22359820269048214,
      "rewards/accuracy_reward": 0.07291666604578495,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4895833358168602,
      "step": 2498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1621.2396240234375,
      "epoch": 0.9597695631301009,
      "grad_norm": 0.1826171875,
      "learning_rate": 1.0437190468172484e-07,
      "loss": 0.0847,
      "num_tokens": 427885689.0,
      "reward": 0.5104166865348816,
      "reward_std": 0.25189625658094883,
      "rewards/accuracy_reward": 0.08333333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1804.2396240234375,
      "epoch": 0.9601536245799328,
      "grad_norm": 0.20703125,
      "learning_rate": 1.0428836694844334e-07,
      "loss": 0.1262,
      "num_tokens": 428075600.0,
      "reward": 0.4010416716337204,
      "reward_std": 0.18002652935683727,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 2500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1603.6250305175781,
      "epoch": 0.9605376860297647,
      "grad_norm": 0.22265625,
      "learning_rate": 1.0420563122531824e-07,
      "loss": 0.1375,
      "num_tokens": 428245956.0,
      "reward": 0.4843750149011612,
      "reward_std": 0.22717511653900146,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.442708320915699,
      "step": 2501
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1562.4896240234375,
      "epoch": 0.9609217474795967,
      "grad_norm": 0.2158203125,
      "learning_rate": 1.0412369766122363e-07,
      "loss": 0.0842,
      "num_tokens": 428412435.0,
      "reward": 0.5442708507180214,
      "reward_std": 0.251308910548687,
      "rewards/accuracy_reward": 0.11458333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 2502
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.6875610351562,
      "epoch": 0.9613058089294287,
      "grad_norm": 0.2451171875,
      "learning_rate": 1.0404256640359008e-07,
      "loss": 0.1436,
      "num_tokens": 428576877.0,
      "reward": 0.4609375223517418,
      "reward_std": 0.18863202538341284,
      "rewards/accuracy_reward": 0.062499999068677425,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2503
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1689.0208740234375,
      "epoch": 0.9616898703792607,
      "grad_norm": 0.21484375,
      "learning_rate": 1.0396223759840458e-07,
      "loss": 0.1229,
      "num_tokens": 428754487.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.232204120606184,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 2504
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1251.8437805175781,
      "epoch": 0.9620739318290926,
      "grad_norm": 0.455078125,
      "learning_rate": 1.038827113902102e-07,
      "loss": 0.1575,
      "num_tokens": 428888464.0,
      "reward": 0.5729166865348816,
      "reward_std": 0.24358141422271729,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.5000000074505806,
      "step": 2505
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1701.2916870117188,
      "epoch": 0.9624579932789247,
      "grad_norm": 0.2265625,
      "learning_rate": 1.0380398792210577e-07,
      "loss": 0.1256,
      "num_tokens": 429066788.0,
      "reward": 0.4921875223517418,
      "reward_std": 0.24612905830144882,
      "rewards/accuracy_reward": 0.08333333302289248,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 2506
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1428.7812805175781,
      "epoch": 0.9628420547287566,
      "grad_norm": 0.2333984375,
      "learning_rate": 1.0372606733574577e-07,
      "loss": 0.0414,
      "num_tokens": 429217431.0,
      "reward": 0.479166679084301,
      "reward_std": 0.1899869730696082,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2507
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1655.9479675292969,
      "epoch": 0.9632261161785886,
      "grad_norm": 0.2216796875,
      "learning_rate": 1.0364894977133984e-07,
      "loss": 0.097,
      "num_tokens": 429389466.0,
      "reward": 0.4973958507180214,
      "reward_std": 0.2775016203522682,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828124925494194,
      "step": 2508
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1514.4062805175781,
      "epoch": 0.9636101776284205,
      "grad_norm": 0.25390625,
      "learning_rate": 1.0357263536765275e-07,
      "loss": 0.0845,
      "num_tokens": 429548889.0,
      "reward": 0.4921875074505806,
      "reward_std": 0.23441363032907248,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041716337204,
      "step": 2509
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.8958740234375,
      "epoch": 0.9639942390782525,
      "grad_norm": 0.1875,
      "learning_rate": 1.0349712426200412e-07,
      "loss": 0.1113,
      "num_tokens": 429733671.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.25575441867113113,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166716337204,
      "step": 2510
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.7709045410156,
      "epoch": 0.9643783005280845,
      "grad_norm": 0.1796875,
      "learning_rate": 1.0342241659026801e-07,
      "loss": 0.0885,
      "num_tokens": 429917049.0,
      "reward": 0.4348958507180214,
      "reward_std": 0.1979926060885191,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 2511
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1674.5937805175781,
      "epoch": 0.9647623619779164,
      "grad_norm": 0.21484375,
      "learning_rate": 1.0334851248687295e-07,
      "loss": 0.1236,
      "num_tokens": 430091458.0,
      "reward": 0.4453125074505806,
      "reward_std": 0.20644613914191723,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 2512
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1793.0625610351562,
      "epoch": 0.9651464234277485,
      "grad_norm": 0.201171875,
      "learning_rate": 1.0327541208480126e-07,
      "loss": 0.1115,
      "num_tokens": 430276984.0,
      "reward": 0.369791679084301,
      "reward_std": 0.1243543028831482,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 2513
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1626.7292175292969,
      "epoch": 0.9655304848775804,
      "grad_norm": 0.201171875,
      "learning_rate": 1.0320311551558935e-07,
      "loss": 0.1176,
      "num_tokens": 430447734.0,
      "reward": 0.5286458432674408,
      "reward_std": 0.25525594875216484,
      "rewards/accuracy_reward": 0.10416666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 2514
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1672.4896240234375,
      "epoch": 0.9659145463274124,
      "grad_norm": 0.23046875,
      "learning_rate": 1.031316229093271e-07,
      "loss": 0.0927,
      "num_tokens": 430625605.0,
      "reward": 0.4088541716337204,
      "reward_std": 0.13389806635677814,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2515
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.8229370117188,
      "epoch": 0.9662986077772444,
      "grad_norm": 0.2353515625,
      "learning_rate": 1.0306093439465781e-07,
      "loss": 0.1048,
      "num_tokens": 430797084.0,
      "reward": 0.5572916716337204,
      "reward_std": 0.32694579288363457,
      "rewards/accuracy_reward": 0.12499999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4322916641831398,
      "step": 2516
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1657.5104675292969,
      "epoch": 0.9666826692270764,
      "grad_norm": 0.2275390625,
      "learning_rate": 1.0299105009877779e-07,
      "loss": 0.0971,
      "num_tokens": 430971045.0,
      "reward": 0.4739583432674408,
      "reward_std": 0.24078267998993397,
      "rewards/accuracy_reward": 0.09374999906867743,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 2517
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1391.2708740234375,
      "epoch": 0.9670667306769083,
      "grad_norm": 0.279296875,
      "learning_rate": 1.0292197014743639e-07,
      "loss": 0.0455,
      "num_tokens": 431118727.0,
      "reward": 0.6328125149011612,
      "reward_std": 0.22434458136558533,
      "rewards/accuracy_reward": 0.16666666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 2518
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1573.4166870117188,
      "epoch": 0.9674507921267402,
      "grad_norm": 0.2431640625,
      "learning_rate": 1.0285369466493553e-07,
      "loss": 0.1134,
      "num_tokens": 431283287.0,
      "reward": 0.502604179084301,
      "reward_std": 0.2888835594058037,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 2519
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1704.5416870117188,
      "epoch": 0.9678348535765723,
      "grad_norm": 0.205078125,
      "learning_rate": 1.0278622377412949e-07,
      "loss": 0.0788,
      "num_tokens": 431459123.0,
      "reward": 0.4322916716337204,
      "reward_std": 0.1519871149212122,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 2520
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.0416870117188,
      "epoch": 0.9682189150264042,
      "grad_norm": 0.2451171875,
      "learning_rate": 1.0271955759642506e-07,
      "loss": 0.0824,
      "num_tokens": 431614879.0,
      "reward": 0.489583358168602,
      "reward_std": 0.21060186624526978,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 2521
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1635.8854370117188,
      "epoch": 0.9686029764762362,
      "grad_norm": 0.216796875,
      "learning_rate": 1.0265369625178078e-07,
      "loss": 0.0815,
      "num_tokens": 431786996.0,
      "reward": 0.4843750298023224,
      "reward_std": 0.2468616757541895,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416716337204,
      "step": 2522
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1613.6354675292969,
      "epoch": 0.9689870379260682,
      "grad_norm": 0.2255859375,
      "learning_rate": 1.0258863985870704e-07,
      "loss": 0.1079,
      "num_tokens": 431954785.0,
      "reward": 0.4973958432674408,
      "reward_std": 0.1923666950315237,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 2523
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1558.8750305175781,
      "epoch": 0.9693710993759002,
      "grad_norm": 0.2216796875,
      "learning_rate": 1.0252438853426586e-07,
      "loss": 0.1204,
      "num_tokens": 432115765.0,
      "reward": 0.591145858168602,
      "reward_std": 0.3469925932586193,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 2524
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1835.2083435058594,
      "epoch": 0.9697551608257321,
      "grad_norm": 0.162109375,
      "learning_rate": 1.024609423940705e-07,
      "loss": 0.1105,
      "num_tokens": 432305889.0,
      "reward": 0.3359375,
      "reward_std": 0.11909882863983512,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3255208358168602,
      "step": 2525
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1828.6979370117188,
      "epoch": 0.970139222275564,
      "grad_norm": 0.171875,
      "learning_rate": 1.0239830155228561e-07,
      "loss": 0.0624,
      "num_tokens": 432496564.0,
      "reward": 0.4375000149011612,
      "reward_std": 0.1941443458199501,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 2526
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.9270935058594,
      "epoch": 0.9705232837253961,
      "grad_norm": 0.1953125,
      "learning_rate": 1.0233646612162657e-07,
      "loss": 0.0891,
      "num_tokens": 432690725.0,
      "reward": 0.3958333432674408,
      "reward_std": 0.16686045564711094,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3541666641831398,
      "step": 2527
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1704.6666870117188,
      "epoch": 0.970907345175228,
      "grad_norm": 0.19921875,
      "learning_rate": 1.0227543621335954e-07,
      "loss": 0.1246,
      "num_tokens": 432868917.0,
      "reward": 0.424479179084301,
      "reward_std": 0.1994890160858631,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3723958358168602,
      "step": 2528
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1753.7500305175781,
      "epoch": 0.97129140662506,
      "grad_norm": 0.208984375,
      "learning_rate": 1.0221521193730138e-07,
      "loss": 0.0906,
      "num_tokens": 433050413.0,
      "reward": 0.4609375149011612,
      "reward_std": 0.22154635936021805,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 2529
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1578.8854675292969,
      "epoch": 0.971675468074892,
      "grad_norm": 0.2138671875,
      "learning_rate": 1.0215579340181912e-07,
      "loss": 0.0629,
      "num_tokens": 433214570.0,
      "reward": 0.5312500074505806,
      "reward_std": 0.20890436694025993,
      "rewards/accuracy_reward": 0.0729166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333358168602,
      "step": 2530
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1708.7916870117188,
      "epoch": 0.972059529524724,
      "grad_norm": 0.2021484375,
      "learning_rate": 1.0209718071383005e-07,
      "loss": 0.0984,
      "num_tokens": 433391638.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.20511537045240402,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541641831398,
      "step": 2531
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1570.2084045410156,
      "epoch": 0.9724435909745559,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.020393739788014e-07,
      "loss": 0.0998,
      "num_tokens": 433555330.0,
      "reward": 0.541666679084301,
      "reward_std": 0.2625061497092247,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4375000074505806,
      "step": 2532
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1538.0312805175781,
      "epoch": 0.972827652424388,
      "grad_norm": 0.2138671875,
      "learning_rate": 1.0198237330075021e-07,
      "loss": 0.0738,
      "num_tokens": 433715517.0,
      "reward": 0.5625000074505806,
      "reward_std": 0.16418463177978992,
      "rewards/accuracy_reward": 0.13541666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833432674408,
      "step": 2533
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1647.4687805175781,
      "epoch": 0.9732117138742199,
      "grad_norm": 0.19921875,
      "learning_rate": 1.0192617878224303e-07,
      "loss": 0.0862,
      "num_tokens": 433889090.0,
      "reward": 0.4817708432674408,
      "reward_std": 0.20483294129371643,
      "rewards/accuracy_reward": 0.08333333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3984375,
      "step": 2534
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1856.6771240234375,
      "epoch": 0.9735957753240518,
      "grad_norm": 0.1669921875,
      "learning_rate": 1.0187079052439585e-07,
      "loss": 0.103,
      "num_tokens": 434081475.0,
      "reward": 0.3984375149011612,
      "reward_std": 0.18080019392073154,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671874925494194,
      "step": 2535
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1571.8021240234375,
      "epoch": 0.9739798367738838,
      "grad_norm": 0.30859375,
      "learning_rate": 1.0181620862687388e-07,
      "loss": 0.121,
      "num_tokens": 434247080.0,
      "reward": 0.5104166865348816,
      "reward_std": 0.20551854744553566,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666641831398,
      "step": 2536
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1557.7708740234375,
      "epoch": 0.9743638982237158,
      "grad_norm": 0.287109375,
      "learning_rate": 1.0176243318789141e-07,
      "loss": 0.1309,
      "num_tokens": 434412690.0,
      "reward": 0.5182291865348816,
      "reward_std": 0.2278377115726471,
      "rewards/accuracy_reward": 0.11458333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4036458358168602,
      "step": 2537
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1523.8958740234375,
      "epoch": 0.9747479596735478,
      "grad_norm": 0.25390625,
      "learning_rate": 1.0170946430421158e-07,
      "loss": 0.0522,
      "num_tokens": 434572632.0,
      "reward": 0.6197916865348816,
      "reward_std": 0.2793177329003811,
      "rewards/accuracy_reward": 0.20833333674818277,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4114583358168602,
      "step": 2538
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1654.8229370117188,
      "epoch": 0.9751320211233797,
      "grad_norm": 0.208984375,
      "learning_rate": 1.0165730207114611e-07,
      "loss": 0.0449,
      "num_tokens": 434745303.0,
      "reward": 0.5416666865348816,
      "reward_std": 0.3150857836008072,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2539
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1328.0312805175781,
      "epoch": 0.9755160825732118,
      "grad_norm": 0.326171875,
      "learning_rate": 1.0160594658255536e-07,
      "loss": 0.1305,
      "num_tokens": 434888970.0,
      "reward": 0.645833358168602,
      "reward_std": 0.3028710335493088,
      "rewards/accuracy_reward": 0.17708333488553762,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.46875,
      "step": 2540
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1612.8438110351562,
      "epoch": 0.9759001440230437,
      "grad_norm": 0.2197265625,
      "learning_rate": 1.0155539793084812e-07,
      "loss": 0.1198,
      "num_tokens": 435058875.0,
      "reward": 0.5520833656191826,
      "reward_std": 0.24986891448497772,
      "rewards/accuracy_reward": 0.12500000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833358168602,
      "step": 2541
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1615.6771240234375,
      "epoch": 0.9762842054728756,
      "grad_norm": 0.224609375,
      "learning_rate": 1.0150565620698107e-07,
      "loss": 0.1281,
      "num_tokens": 435227340.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.129163334146142,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791567325592,
      "step": 2542
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1465.0938110351562,
      "epoch": 0.9766682669227076,
      "grad_norm": 0.236328125,
      "learning_rate": 1.0145672150045923e-07,
      "loss": 0.0419,
      "num_tokens": 435381437.0,
      "reward": 0.7604166865348816,
      "reward_std": 0.3282424882054329,
      "rewards/accuracy_reward": 0.2812499962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4791666641831398,
      "step": 2543
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1498.7187805175781,
      "epoch": 0.9770523283725396,
      "grad_norm": 0.2158203125,
      "learning_rate": 1.0140859389933528e-07,
      "loss": 0.0433,
      "num_tokens": 435540282.0,
      "reward": 0.5911458507180214,
      "reward_std": 0.2944330610334873,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4765625,
      "step": 2544
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1501.2083740234375,
      "epoch": 0.9774363898223716,
      "grad_norm": 0.2373046875,
      "learning_rate": 1.013612734902097e-07,
      "loss": 0.0631,
      "num_tokens": 435699270.0,
      "reward": 0.5234375149011612,
      "reward_std": 0.2481831405311823,
      "rewards/accuracy_reward": 0.09375000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4296875,
      "step": 2545
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1649.0000610351562,
      "epoch": 0.9778204512722035,
      "grad_norm": 0.1826171875,
      "learning_rate": 1.013147603582305e-07,
      "loss": 0.1041,
      "num_tokens": 435871038.0,
      "reward": 0.5052083358168602,
      "reward_std": 0.18436023127287626,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.421875,
      "step": 2546
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1651.7292175292969,
      "epoch": 0.9782045127220356,
      "grad_norm": 0.1865234375,
      "learning_rate": 1.0126905458709296e-07,
      "loss": 0.019,
      "num_tokens": 436044116.0,
      "reward": 0.4843750223517418,
      "reward_std": 0.19173270463943481,
      "rewards/accuracy_reward": 0.0833333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4010416641831398,
      "step": 2547
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1688.9063110351562,
      "epoch": 0.9785885741718675,
      "grad_norm": 0.197265625,
      "learning_rate": 1.0122415625903978e-07,
      "loss": 0.0484,
      "num_tokens": 436219595.0,
      "reward": 0.4479166716337204,
      "reward_std": 0.21506720781326294,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3958333358168602,
      "step": 2548
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1751.8854370117188,
      "epoch": 0.9789726356216994,
      "grad_norm": 0.1826171875,
      "learning_rate": 1.0118006545486067e-07,
      "loss": 0.092,
      "num_tokens": 436401080.0,
      "reward": 0.4036458432674408,
      "reward_std": 0.1708046142011881,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 2549
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1727.5104675292969,
      "epoch": 0.9793566970715314,
      "grad_norm": 0.201171875,
      "learning_rate": 1.0113678225389225e-07,
      "loss": 0.1129,
      "num_tokens": 436581121.0,
      "reward": 0.471354179084301,
      "reward_std": 0.19471271708607674,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 2550
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1746.6979370117188,
      "epoch": 0.9797407585213634,
      "grad_norm": 0.201171875,
      "learning_rate": 1.01094306734018e-07,
      "loss": 0.1152,
      "num_tokens": 436764940.0,
      "reward": 0.3593750074505806,
      "reward_std": 0.12082788720726967,
      "rewards/accuracy_reward": 0.0,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 2551
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1405.9375610351562,
      "epoch": 0.9801248199711954,
      "grad_norm": 0.275390625,
      "learning_rate": 1.0105263897166807e-07,
      "loss": 0.1295,
      "num_tokens": 436914142.0,
      "reward": 0.5572916939854622,
      "reward_std": 0.1868427861481905,
      "rewards/accuracy_reward": 0.1354166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218749925494194,
      "step": 2552
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1625.6667175292969,
      "epoch": 0.9805088814210273,
      "grad_norm": 0.216796875,
      "learning_rate": 1.0101177904181914e-07,
      "loss": 0.1131,
      "num_tokens": 437083934.0,
      "reward": 0.5078125074505806,
      "reward_std": 0.23112766817212105,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 2553
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1781.7188110351562,
      "epoch": 0.9808929428708594,
      "grad_norm": 0.1806640625,
      "learning_rate": 1.0097172701799422e-07,
      "loss": 0.083,
      "num_tokens": 437268427.0,
      "reward": 0.4218750074505806,
      "reward_std": 0.195728100836277,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083358168602,
      "step": 2554
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1632.0416870117188,
      "epoch": 0.9812770043206913,
      "grad_norm": 0.2041015625,
      "learning_rate": 1.009324829722626e-07,
      "loss": 0.1239,
      "num_tokens": 437438863.0,
      "reward": 0.4765625074505806,
      "reward_std": 0.16626954823732376,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 2555
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1427.0000457763672,
      "epoch": 0.9816610657705233,
      "grad_norm": 0.287109375,
      "learning_rate": 1.0089404697523981e-07,
      "loss": 0.0752,
      "num_tokens": 437590039.0,
      "reward": 0.6250000149011612,
      "reward_std": 0.22882095351815224,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333432674408,
      "step": 2556
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1594.9271240234375,
      "epoch": 0.9820451272203553,
      "grad_norm": 0.232421875,
      "learning_rate": 1.0085641909608719e-07,
      "loss": 0.0798,
      "num_tokens": 437758152.0,
      "reward": 0.575520858168602,
      "reward_std": 0.267150841653347,
      "rewards/accuracy_reward": 0.16666666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4088541716337204,
      "step": 2557
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1528.1041870117188,
      "epoch": 0.9824291886701872,
      "grad_norm": 0.2099609375,
      "learning_rate": 1.008195994025121e-07,
      "loss": 0.0887,
      "num_tokens": 437919426.0,
      "reward": 0.5546875074505806,
      "reward_std": 0.28054073452949524,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4401041641831398,
      "step": 2558
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1459.3854370117188,
      "epoch": 0.9828132501200192,
      "grad_norm": 0.26171875,
      "learning_rate": 1.007835879607676e-07,
      "loss": 0.0648,
      "num_tokens": 438072519.0,
      "reward": 0.6510416865348816,
      "reward_std": 0.2586767878383398,
      "rewards/accuracy_reward": 0.1979166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4531250074505806,
      "step": 2559
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1345.9271240234375,
      "epoch": 0.9831973115698511,
      "grad_norm": 0.236328125,
      "learning_rate": 1.007483848356525e-07,
      "loss": 0.0949,
      "num_tokens": 438213736.0,
      "reward": 0.5156250074505806,
      "reward_std": 0.15792139247059822,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4739583358168602,
      "step": 2560
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1678.2708740234375,
      "epoch": 0.9835813730196832,
      "grad_norm": 0.2216796875,
      "learning_rate": 1.007139900905109e-07,
      "loss": 0.1041,
      "num_tokens": 438388186.0,
      "reward": 0.4843750074505806,
      "reward_std": 0.2423115335404873,
      "rewards/accuracy_reward": 0.06250000093132257,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4218750074505806,
      "step": 2561
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1638.9479675292969,
      "epoch": 0.9839654344695151,
      "grad_norm": 0.212890625,
      "learning_rate": 1.0068040378723256e-07,
      "loss": 0.0936,
      "num_tokens": 438564109.0,
      "reward": 0.5390625149011612,
      "reward_std": 0.2070658579468727,
      "rewards/accuracy_reward": 0.15625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828124925494194,
      "step": 2562
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1518.5313110351562,
      "epoch": 0.9843494959193471,
      "grad_norm": 0.234375,
      "learning_rate": 1.0064762598625237e-07,
      "loss": 0.086,
      "num_tokens": 438722120.0,
      "reward": 0.606770858168602,
      "reward_std": 0.2624484524130821,
      "rewards/accuracy_reward": 0.13541666325181723,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4713541567325592,
      "step": 2563
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1547.1667175292969,
      "epoch": 0.9847335573691791,
      "grad_norm": 0.2470703125,
      "learning_rate": 1.0061565674655048e-07,
      "loss": 0.0767,
      "num_tokens": 438884656.0,
      "reward": 0.5000000149011612,
      "reward_std": 0.21469112485647202,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4583333283662796,
      "step": 2564
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1506.9062805175781,
      "epoch": 0.985117618819011,
      "grad_norm": 0.2578125,
      "learning_rate": 1.0058449612565209e-07,
      "loss": 0.0638,
      "num_tokens": 439044679.0,
      "reward": 0.5390625,
      "reward_std": 0.22126594185829163,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4661458358168602,
      "step": 2565
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1349.9687805175781,
      "epoch": 0.985501680268843,
      "grad_norm": 0.2412109375,
      "learning_rate": 1.0055414417962737e-07,
      "loss": 0.0876,
      "num_tokens": 439187908.0,
      "reward": 0.630208358168602,
      "reward_std": 0.25971841625869274,
      "rewards/accuracy_reward": 0.1666666641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416716337204,
      "step": 2566
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1739.3021240234375,
      "epoch": 0.9858857417186749,
      "grad_norm": 0.1875,
      "learning_rate": 1.0052460096309138e-07,
      "loss": 0.0563,
      "num_tokens": 439369081.0,
      "reward": 0.4348958432674408,
      "reward_std": 0.17999800108373165,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625074505806,
      "step": 2567
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1767.0729370117188,
      "epoch": 0.986269803168507,
      "grad_norm": 0.1904296875,
      "learning_rate": 1.0049586652920398e-07,
      "loss": 0.0979,
      "num_tokens": 439552536.0,
      "reward": 0.3932291716337204,
      "reward_std": 0.13963515125215054,
      "rewards/accuracy_reward": 0.03125,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 2568
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1592.6250610351562,
      "epoch": 0.9866538646183389,
      "grad_norm": 0.25,
      "learning_rate": 1.0046794092966968e-07,
      "loss": 0.07,
      "num_tokens": 439720348.0,
      "reward": 0.5625000223517418,
      "reward_std": 0.3137785103172064,
      "rewards/accuracy_reward": 0.15625000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 2569
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1569.6667175292969,
      "epoch": 0.9870379260681709,
      "grad_norm": 0.236328125,
      "learning_rate": 1.0044082421473765e-07,
      "loss": 0.127,
      "num_tokens": 439887956.0,
      "reward": 0.4895833507180214,
      "reward_std": 0.2097405195236206,
      "rewards/accuracy_reward": 0.1041666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3854166641831398,
      "step": 2570
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1566.3854370117188,
      "epoch": 0.9874219875180029,
      "grad_norm": 0.228515625,
      "learning_rate": 1.0041451643320143e-07,
      "loss": 0.0804,
      "num_tokens": 440052465.0,
      "reward": 0.5703125223517418,
      "reward_std": 0.29667652025818825,
      "rewards/accuracy_reward": 0.1354166679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 2571
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1533.3541870117188,
      "epoch": 0.9878060489678349,
      "grad_norm": 0.2490234375,
      "learning_rate": 1.0038901763239904e-07,
      "loss": 0.0805,
      "num_tokens": 440214435.0,
      "reward": 0.4453125149011612,
      "reward_std": 0.18344523012638092,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4244791641831398,
      "step": 2572
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1394.7395935058594,
      "epoch": 0.9881901104176668,
      "grad_norm": 0.29296875,
      "learning_rate": 1.0036432785821286e-07,
      "loss": 0.0903,
      "num_tokens": 440363066.0,
      "reward": 0.5911458507180214,
      "reward_std": 0.28425207547843456,
      "rewards/accuracy_reward": 0.1562499962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4348958358168602,
      "step": 2573
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1716.3125305175781,
      "epoch": 0.9885741718674989,
      "grad_norm": 0.18359375,
      "learning_rate": 1.0034044715506955e-07,
      "loss": 0.0747,
      "num_tokens": 440540144.0,
      "reward": 0.4296875074505806,
      "reward_std": 0.17695098370313644,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2574
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1294.9583435058594,
      "epoch": 0.9889582333173308,
      "grad_norm": 0.244140625,
      "learning_rate": 1.0031737556593972e-07,
      "loss": 0.0918,
      "num_tokens": 440677196.0,
      "reward": 0.502604179084301,
      "reward_std": 0.19283486157655716,
      "rewards/accuracy_reward": 0.041666666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4609375,
      "step": 2575
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1519.6979675292969,
      "epoch": 0.9893422947671627,
      "grad_norm": 0.240234375,
      "learning_rate": 1.0029511313233838e-07,
      "loss": 0.0712,
      "num_tokens": 440839079.0,
      "reward": 0.5651041865348816,
      "reward_std": 0.2325502522289753,
      "rewards/accuracy_reward": 0.11458333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208358168602,
      "step": 2576
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1744.4375305175781,
      "epoch": 0.9897263562169947,
      "grad_norm": 0.171875,
      "learning_rate": 1.0027365989432422e-07,
      "loss": 0.0896,
      "num_tokens": 441019465.0,
      "reward": 0.4375000074505806,
      "reward_std": 0.1646699057891965,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.375,
      "step": 2577
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1550.9062805175781,
      "epoch": 0.9901104176668267,
      "grad_norm": 0.2265625,
      "learning_rate": 1.0025301589050017e-07,
      "loss": 0.1237,
      "num_tokens": 441185584.0,
      "reward": 0.479166679084301,
      "reward_std": 0.21253520622849464,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4166666716337204,
      "step": 2578
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1486.1458740234375,
      "epoch": 0.9904944791166587,
      "grad_norm": 0.259765625,
      "learning_rate": 1.0023318115801282e-07,
      "loss": 0.1438,
      "num_tokens": 441344702.0,
      "reward": 0.5390625223517418,
      "reward_std": 0.25001129508018494,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 2579
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1713.2812805175781,
      "epoch": 0.9908785405664906,
      "grad_norm": 0.201171875,
      "learning_rate": 1.0021415573255265e-07,
      "loss": 0.1406,
      "num_tokens": 441522337.0,
      "reward": 0.3984375149011612,
      "reward_std": 0.15390780195593834,
      "rewards/accuracy_reward": 0.010416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2580
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1757.2396545410156,
      "epoch": 0.9912626020163227,
      "grad_norm": 0.1845703125,
      "learning_rate": 1.001959396483539e-07,
      "loss": 0.0821,
      "num_tokens": 441707664.0,
      "reward": 0.4296875149011612,
      "reward_std": 0.2010653978213668,
      "rewards/accuracy_reward": 0.06250000186264515,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3671875,
      "step": 2581
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1673.2396240234375,
      "epoch": 0.9916466634661546,
      "grad_norm": 0.21875,
      "learning_rate": 1.0017853293819445e-07,
      "loss": 0.0827,
      "num_tokens": 441884263.0,
      "reward": 0.479166679084301,
      "reward_std": 0.2088695652782917,
      "rewards/accuracy_reward": 0.05208333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4270833283662796,
      "step": 2582
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1488.5833740234375,
      "epoch": 0.9920307249159865,
      "grad_norm": 0.26171875,
      "learning_rate": 1.0016193563339571e-07,
      "loss": 0.1328,
      "num_tokens": 442040191.0,
      "reward": 0.4765625149011612,
      "reward_std": 0.22500889375805855,
      "rewards/accuracy_reward": 0.031250000931322575,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 2583
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.6250610351562,
      "epoch": 0.9924147863658185,
      "grad_norm": 0.240234375,
      "learning_rate": 1.0014614776382289e-07,
      "loss": 0.1287,
      "num_tokens": 442205587.0,
      "reward": 0.5234375,
      "reward_std": 0.24247609078884125,
      "rewards/accuracy_reward": 0.10416666977107525,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4192708358168602,
      "step": 2584
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1821.8229675292969,
      "epoch": 0.9927988478156505,
      "grad_norm": 0.1572265625,
      "learning_rate": 1.0013116935788447e-07,
      "loss": 0.0648,
      "num_tokens": 442395418.0,
      "reward": 0.393229179084301,
      "reward_std": 0.13825709279626608,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3515625,
      "step": 2585
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1520.6146240234375,
      "epoch": 0.9931829092654825,
      "grad_norm": 0.2373046875,
      "learning_rate": 1.0011700044253248e-07,
      "loss": 0.1004,
      "num_tokens": 442554773.0,
      "reward": 0.7005208432674408,
      "reward_std": 0.32333657890558243,
      "rewards/accuracy_reward": 0.2499999962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4505208283662796,
      "step": 2586
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1773.5000305175781,
      "epoch": 0.9935669707153144,
      "grad_norm": 0.1875,
      "learning_rate": 1.001036410432624e-07,
      "loss": 0.0812,
      "num_tokens": 442740269.0,
      "reward": 0.3828125149011612,
      "reward_std": 0.1546316589228809,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3619791716337204,
      "step": 2587
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1698.2709045410156,
      "epoch": 0.9939510321651465,
      "grad_norm": 0.2197265625,
      "learning_rate": 1.00091091184113e-07,
      "loss": 0.045,
      "num_tokens": 442916639.0,
      "reward": 0.5442708432674408,
      "reward_std": 0.24463598430156708,
      "rewards/accuracy_reward": 0.1666666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3776041641831398,
      "step": 2588
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1564.9791870117188,
      "epoch": 0.9943350936149784,
      "grad_norm": 0.2216796875,
      "learning_rate": 1.0007935088766645e-07,
      "loss": 0.1012,
      "num_tokens": 443079829.0,
      "reward": 0.4791666939854622,
      "reward_std": 0.18196381255984306,
      "rewards/accuracy_reward": 0.02083333395421505,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.458333320915699,
      "step": 2589
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1516.0729675292969,
      "epoch": 0.9947191550648103,
      "grad_norm": 0.2099609375,
      "learning_rate": 1.000684201750481e-07,
      "loss": 0.017,
      "num_tokens": 443239724.0,
      "reward": 0.6197916865348816,
      "reward_std": 0.292665958404541,
      "rewards/accuracy_reward": 0.1562499962747097,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416641831398,
      "step": 2590
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1464.0938110351562,
      "epoch": 0.9951032165146424,
      "grad_norm": 0.1904296875,
      "learning_rate": 1.0005829906592665e-07,
      "loss": 0.0822,
      "num_tokens": 443394909.0,
      "reward": 0.5390625149011612,
      "reward_std": 0.16958150919526815,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 2591
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1544.4687805175781,
      "epoch": 0.9954872779644743,
      "grad_norm": 0.232421875,
      "learning_rate": 1.0004898757851387e-07,
      "loss": 0.0595,
      "num_tokens": 443555002.0,
      "reward": 0.541666679084301,
      "reward_std": 0.25344638526439667,
      "rewards/accuracy_reward": 0.09375,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166641831398,
      "step": 2592
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1741.9063110351562,
      "epoch": 0.9958713394143063,
      "grad_norm": 0.1943359375,
      "learning_rate": 1.0004048572956483e-07,
      "loss": 0.0866,
      "num_tokens": 443738001.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.2417571358382702,
      "rewards/accuracy_reward": 0.0833333358168602,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3828125,
      "step": 2593
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1560.8854370117188,
      "epoch": 0.9962554008641382,
      "grad_norm": 0.177734375,
      "learning_rate": 1.0003279353437768e-07,
      "loss": 0.0085,
      "num_tokens": 443900790.0,
      "reward": 0.5208333432674408,
      "reward_std": 0.16088383551687002,
      "rewards/accuracy_reward": 0.0729166641831398,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4479166716337204,
      "step": 2594
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1800.2083740234375,
      "epoch": 0.9966394623139703,
      "grad_norm": 0.1474609375,
      "learning_rate": 1.0002591100679366e-07,
      "loss": 0.0455,
      "num_tokens": 444088434.0,
      "reward": 0.4531250074505806,
      "reward_std": 0.15277442056685686,
      "rewards/accuracy_reward": 0.09374999813735485,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.359375,
      "step": 2595
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1711.2292175292969,
      "epoch": 0.9970235237638022,
      "grad_norm": 0.2021484375,
      "learning_rate": 1.0001983815919722e-07,
      "loss": 0.0719,
      "num_tokens": 444269704.0,
      "reward": 0.4427083507180214,
      "reward_std": 0.18732366524636745,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3802083283662796,
      "step": 2596
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1505.9167175292969,
      "epoch": 0.9974075852136342,
      "grad_norm": 0.2333984375,
      "learning_rate": 1.0001457500251575e-07,
      "loss": 0.1596,
      "num_tokens": 444426640.0,
      "reward": 0.6093750074505806,
      "reward_std": 0.34603866934776306,
      "rewards/accuracy_reward": 0.1458333320915699,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4635416641831398,
      "step": 2597
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1637.0937805175781,
      "epoch": 0.9977916466634662,
      "grad_norm": 0.2109375,
      "learning_rate": 1.0001012154621975e-07,
      "loss": 0.1248,
      "num_tokens": 444598817.0,
      "reward": 0.4869791865348816,
      "reward_std": 0.2144615612924099,
      "rewards/accuracy_reward": 0.0416666679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4453125,
      "step": 2598
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1548.1562805175781,
      "epoch": 0.9981757081132981,
      "grad_norm": 0.2431640625,
      "learning_rate": 1.0000647779832274e-07,
      "loss": 0.0371,
      "num_tokens": 444765584.0,
      "reward": 0.486979179084301,
      "reward_std": 0.23280653171241283,
      "rewards/accuracy_reward": 0.07291666697710752,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4140625,
      "step": 2599
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1660.4479675292969,
      "epoch": 0.9985597695631301,
      "grad_norm": 0.177734375,
      "learning_rate": 1.0000364376538117e-07,
      "loss": 0.1202,
      "num_tokens": 444941115.0,
      "reward": 0.4661458432674408,
      "reward_std": 0.1786599401384592,
      "rewards/accuracy_reward": 0.0885416679084301,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3932291641831398,
      "step": 2600
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1634.2292175292969,
      "epoch": 0.998943831012962,
      "grad_norm": 0.20703125,
      "learning_rate": 1.0000161945249471e-07,
      "loss": 0.058,
      "num_tokens": 445111745.0,
      "reward": 0.4687500149011612,
      "reward_std": 0.20727644860744476,
      "rewards/accuracy_reward": 0.0625,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.40625,
      "step": 2601
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1575.5833740234375,
      "epoch": 0.9993278924627941,
      "grad_norm": 0.2353515625,
      "learning_rate": 1.0000040486330578e-07,
      "loss": 0.1407,
      "num_tokens": 445276009.0,
      "reward": 0.6067708432674408,
      "reward_std": 0.2665208652615547,
      "rewards/accuracy_reward": 0.2187500074505806,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.3880208358168602,
      "step": 2602
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1552.9125366210938,
      "epoch": 0.999711953912626,
      "grad_norm": 0.21875,
      "learning_rate": 1e-07,
      "loss": 0.0897,
      "num_tokens": 445442264.0,
      "reward": 0.4947916865348816,
      "reward_std": 0.21028348803520203,
      "rewards/accuracy_reward": 0.052083334885537624,
      "rewards/format_reward": 0.0,
      "rewards/tag_count_reward": 0.4427083358168602,
      "step": 2603
    },
    {
      "epoch": 0.999711953912626,
      "step": 2603,
      "total_flos": 0.0,
      "train_loss": 0.09429229835021126,
      "train_runtime": 142210.8778,
      "train_samples_per_second": 0.22,
      "train_steps_per_second": 0.018
    }
  ],
  "logging_steps": 1,
  "max_steps": 2603,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 200,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}