zerozeroz/CodeLlama-7b-hf_1000rl_cpp_javascript
028
1{2 "best_metric": null,3 "best_model_checkpoint": null,4 "epoch": 3.0,5 "eval_steps": 500,6 "global_step": 228,7 "is_hyper_param_search": false,8 "is_local_process_zero": true,9 "is_world_process_zero": true,10 "log_history": [11 {12 "completion_length": 99.10416793823242,13 "epoch": 0.013157894736842105,14 "grad_norm": 1.1334748507973709,15 "kl": 0.0,16 "learning_rate": 7.142857142857142e-08,17 "loss": -0.0,18 "reward": 0.6663198471069336,19 "reward_std": 0.4498612582683563,20 "rewards/correct_code_reward_func": 0.2500000149011612,21 "rewards/len_reward_func": 0.4163198471069336,22 "step": 123 },24 {25 "completion_length": 91.58333587646484,26 "epoch": 0.02631578947368421,27 "grad_norm": 16.060906203241963,28 "kl": 0.0,29 "learning_rate": 1.4285714285714285e-07,30 "loss": -0.0,31 "reward": 0.9228171110153198,32 "reward_std": 0.4594407230615616,33 "rewards/correct_code_reward_func": 0.5416666865348816,34 "rewards/len_reward_func": 0.38115040957927704,35 "step": 236 },37 {38 "completion_length": 72.02083587646484,39 "epoch": 0.039473684210526314,40 "grad_norm": 3.1753635339709736,41 "kl": 0.00012445449829101562,42 "learning_rate": 2.1428571428571426e-07,43 "loss": 0.0,44 "reward": 0.6041666865348816,45 "reward_std": 0.25199543684720993,46 "rewards/correct_code_reward_func": 0.1666666679084301,47 "rewards/len_reward_func": 0.4375000149011612,48 "step": 349 },50 {51 "completion_length": 87.33333396911621,52 "epoch": 0.05263157894736842,53 "grad_norm": 5.760299497142578,54 "kl": -1.5407800674438477e-05,55 "learning_rate": 2.857142857142857e-07,56 "loss": -0.0,57 "reward": 0.7341760993003845,58 "reward_std": 0.37608518451452255,59 "rewards/correct_code_reward_func": 0.31250002048909664,60 "rewards/len_reward_func": 0.4216760843992233,61 "step": 462 },63 {64 "completion_length": 71.85416793823242,65 "epoch": 0.06578947368421052,66 "grad_norm": 12.020598074533623,67 "kl": 0.00014781951904296875,68 "learning_rate": 3.5714285714285716e-07,69 "loss": 0.0,70 "reward": 0.5998998582363129,71 "reward_std": 0.20088782534003258,72 "rewards/correct_code_reward_func": 0.14583333395421505,73 "rewards/len_reward_func": 0.45406651496887207,74 "step": 575 },76 {77 "completion_length": 89.77083587646484,78 "epoch": 0.07894736842105263,79 "grad_norm": 0.8550201889336397,80 "kl": 4.374980926513672e-05,81 "learning_rate": 4.285714285714285e-07,82 "loss": 0.0,83 "reward": 0.5605520606040955,84 "reward_std": 0.2179059162735939,85 "rewards/correct_code_reward_func": 0.1041666679084301,86 "rewards/len_reward_func": 0.45638537406921387,87 "step": 688 },89 {90 "completion_length": 72.79167175292969,91 "epoch": 0.09210526315789473,92 "grad_norm": 3.0170118497826293,93 "kl": 0.00016558170318603516,94 "learning_rate": 5e-07,95 "loss": 0.0,96 "reward": 0.9927114844322205,97 "reward_std": 0.40002472698688507,98 "rewards/correct_code_reward_func": 0.5625,99 "rewards/len_reward_func": 0.4302114397287369,100 "step": 7101 },102 {103 "completion_length": 81.66667175292969,104 "epoch": 0.10526315789473684,105 "grad_norm": 4.916856440024865,106 "kl": 7.331371307373047e-05,107 "learning_rate": 4.999747408985249e-07,108 "loss": 0.0,109 "reward": 0.9147771000862122,110 "reward_std": 0.43089577555656433,111 "rewards/correct_code_reward_func": 0.5208333432674408,112 "rewards/len_reward_func": 0.39394378662109375,113 "step": 8114 },115 {116 "completion_length": 106.1875057220459,117 "epoch": 0.11842105263157894,118 "grad_norm": 7.051099519560641,119 "kl": 0.00026702880859375,120 "learning_rate": 4.998989686982771e-07,121 "loss": 0.0,122 "reward": 0.9007994532585144,123 "reward_std": 0.30253712832927704,124 "rewards/correct_code_reward_func": 0.4791666716337204,125 "rewards/len_reward_func": 0.4216327518224716,126 "step": 9127 },128 {129 "completion_length": 89.25,130 "epoch": 0.13157894736842105,131 "grad_norm": 3.0953263442901586,132 "kl": 0.00014472007751464844,133 "learning_rate": 4.997726987107581e-07,134 "loss": 0.0,135 "reward": 0.8582921326160431,136 "reward_std": 0.4781641513109207,137 "rewards/correct_code_reward_func": 0.4166666716337204,138 "rewards/len_reward_func": 0.4416254311800003,139 "step": 10140 },141 {142 "completion_length": 61.0625,143 "epoch": 0.14473684210526316,144 "grad_norm": 26.321796409287952,145 "kl": 0.00046253204345703125,146 "learning_rate": 4.995959564516996e-07,147 "loss": 0.0,148 "reward": 1.1543937921524048,149 "reward_std": 0.46843117475509644,150 "rewards/correct_code_reward_func": 0.7708333432674408,151 "rewards/len_reward_func": 0.3835604339838028,152 "step": 11153 },154 {155 "completion_length": 57.687503814697266,156 "epoch": 0.15789473684210525,157 "grad_norm": 2.6899417118195452,158 "kl": 0.000301361083984375,159 "learning_rate": 4.993687776359066e-07,160 "loss": 0.0,161 "reward": 0.8750000298023224,162 "reward_std": 0.37565621733665466,163 "rewards/correct_code_reward_func": 0.395833358168602,164 "rewards/len_reward_func": 0.4791666716337204,165 "step": 12166 },167 {168 "completion_length": 160.35416793823242,169 "epoch": 0.17105263157894737,170 "grad_norm": 2.247154751130096,171 "kl": 0.0006351470947265625,172 "learning_rate": 4.990912081700413e-07,173 "loss": 0.0,174 "reward": 0.5879067778587341,175 "reward_std": 0.3584153801202774,176 "rewards/correct_code_reward_func": 0.2291666679084301,177 "rewards/len_reward_func": 0.3587401211261749,178 "step": 13179 },180 {181 "completion_length": 54.916669845581055,182 "epoch": 0.18421052631578946,183 "grad_norm": 7.0002606987315685,184 "kl": 0.000507354736328125,185 "learning_rate": 4.987633041433461e-07,186 "loss": 0.0,187 "reward": 0.9894838631153107,188 "reward_std": 0.5111564844846725,189 "rewards/correct_code_reward_func": 0.5416666716337204,190 "rewards/len_reward_func": 0.4478171765804291,191 "step": 14192 },193 {194 "completion_length": 63.06250190734863,195 "epoch": 0.19736842105263158,196 "grad_norm": 2.6443252242541284,197 "kl": 0.001739501953125,198 "learning_rate": 4.983851318163097e-07,199 "loss": 0.0,200 "reward": 0.8326185643672943,201 "reward_std": 0.29923243820667267,202 "rewards/correct_code_reward_func": 0.3958333432674408,203 "rewards/len_reward_func": 0.4367852360010147,204 "step": 15205 },206 {207 "completion_length": 99.39583587646484,208 "epoch": 0.21052631578947367,209 "grad_norm": 1.0079451181922614,210 "kl": 0.00141143798828125,211 "learning_rate": 4.979567676072775e-07,212 "loss": 0.0,213 "reward": 0.852995753288269,214 "reward_std": 0.34659716486930847,215 "rewards/correct_code_reward_func": 0.3958333432674408,216 "rewards/len_reward_func": 0.45716238021850586,217 "step": 16218 },219 {220 "completion_length": 53.08333396911621,221 "epoch": 0.2236842105263158,222 "grad_norm": 9.967406022456908,223 "kl": 0.0040435791015625,224 "learning_rate": 4.9747829807701e-07,225 "loss": 0.0,226 "reward": 0.8658536076545715,227 "reward_std": 0.426578089594841,228 "rewards/correct_code_reward_func": 0.4166666865348816,229 "rewards/len_reward_func": 0.44918693602085114,230 "step": 17231 },232 {233 "completion_length": 143.7916717529297,234 "epoch": 0.23684210526315788,235 "grad_norm": 1.9448270150109064,236 "kl": 0.001434326171875,237 "learning_rate": 4.9694981991119e-07,238 "loss": 0.0,239 "reward": 0.7298745512962341,240 "reward_std": 0.28270113095641136,241 "rewards/correct_code_reward_func": 0.3333333432674408,242 "rewards/len_reward_func": 0.39654120802879333,243 "step": 18244 },245 {246 "completion_length": 61.041669845581055,247 "epoch": 0.25,248 "grad_norm": 3.1720724842373182,249 "kl": 0.00389862060546875,250 "learning_rate": 4.963714399008868e-07,251 "loss": 0.0,252 "reward": 0.718452662229538,253 "reward_std": 0.37869784235954285,254 "rewards/correct_code_reward_func": 0.2500000149011612,255 "rewards/len_reward_func": 0.46845264732837677,256 "step": 19257 },258 {259 "completion_length": 79.89583587646484,260 "epoch": 0.2631578947368421,261 "grad_norm": 1.7632061082251036,262 "kl": 0.00446319580078125,263 "learning_rate": 4.957432749209755e-07,264 "loss": 0.0,265 "reward": 0.7348142862319946,266 "reward_std": 0.1320388102903962,267 "rewards/correct_code_reward_func": 0.25,268 "rewards/len_reward_func": 0.48481428623199463,269 "step": 20270 },271 {272 "completion_length": 81.18750381469727,273 "epoch": 0.27631578947368424,274 "grad_norm": 2.5858288436684593,275 "kl": 0.00498199462890625,276 "learning_rate": 4.950654519065195e-07,277 "loss": 0.0,278 "reward": 0.6811683177947998,279 "reward_std": 0.1932540312409401,280 "rewards/correct_code_reward_func": 0.2083333432674408,281 "rewards/len_reward_func": 0.472834974527359,282 "step": 21283 },284 {285 "completion_length": 87.56250381469727,286 "epoch": 0.2894736842105263,287 "grad_norm": 18.2585507937141,288 "kl": 0.03173828125,289 "learning_rate": 4.943381078271214e-07,290 "loss": 0.0,291 "reward": 0.6875,292 "reward_std": 0.28126099705696106,293 "rewards/correct_code_reward_func": 0.2083333432674408,294 "rewards/len_reward_func": 0.4791666716337204,295 "step": 22296 },297 {298 "completion_length": 134.64583587646484,299 "epoch": 0.3026315789473684,300 "grad_norm": 17.07507069634021,301 "kl": 0.12744140625,302 "learning_rate": 4.935613896592445e-07,303 "loss": 0.0001,304 "reward": 0.9976747632026672,305 "reward_std": 0.22606945782899857,306 "rewards/correct_code_reward_func": 0.5416666865348816,307 "rewards/len_reward_func": 0.45600807666778564,308 "step": 23309 },310 {311 "completion_length": 52.25000190734863,312 "epoch": 0.3157894736842105,313 "grad_norm": 5.933528514518437,314 "kl": 0.05426025390625,315 "learning_rate": 4.92735454356513e-07,316 "loss": 0.0001,317 "reward": 0.7500000298023224,318 "reward_std": 0.2994871214032173,319 "rewards/correct_code_reward_func": 0.2708333432674408,320 "rewards/len_reward_func": 0.4791666716337204,321 "step": 24322 },323 {324 "completion_length": 96.29167175292969,325 "epoch": 0.32894736842105265,326 "grad_norm": 0.7449673513689808,327 "kl": 0.0208740234375,328 "learning_rate": 4.918604688179958e-07,329 "loss": 0.0,330 "reward": 0.5824915617704391,331 "reward_std": 0.1475645862519741,332 "rewards/correct_code_reward_func": 0.1041666716337204,333 "rewards/len_reward_func": 0.47832491993904114,334 "step": 25335 },336 {337 "completion_length": 57.04166793823242,338 "epoch": 0.34210526315789475,339 "grad_norm": 7.282800644316476,340 "kl": 0.05474853515625,341 "learning_rate": 4.90936609854481e-07,342 "loss": 0.0001,343 "reward": 1.0208333730697632,344 "reward_std": 0.4474997818470001,345 "rewards/correct_code_reward_func": 0.5416666865348816,346 "rewards/len_reward_func": 0.4791666716337204,347 "step": 26348 },349 {350 "completion_length": 69.29166984558105,351 "epoch": 0.35526315789473684,352 "grad_norm": 11.501413945029437,353 "kl": 0.037353515625,354 "learning_rate": 4.89964064152747e-07,355 "loss": 0.0,356 "reward": 0.8958333432674408,357 "reward_std": 0.2730504274368286,358 "rewards/correct_code_reward_func": 0.4166666716337204,359 "rewards/len_reward_func": 0.4791666716337204,360 "step": 27361 },362 {363 "completion_length": 52.708335876464844,364 "epoch": 0.3684210526315789,365 "grad_norm": 1.4375830290890173,366 "kl": 0.05767822265625,367 "learning_rate": 4.889430282378384e-07,368 "loss": 0.0001,369 "reward": 0.7291666865348816,370 "reward_std": 0.21322893351316452,371 "rewards/correct_code_reward_func": 0.2708333432674408,372 "rewards/len_reward_func": 0.4583333432674408,373 "step": 28374 },375 {376 "completion_length": 84.875,377 "epoch": 0.3815789473684211,378 "grad_norm": 1.8947890037281183,379 "kl": 0.079345703125,380 "learning_rate": 4.878737084333535e-07,381 "loss": 0.0001,382 "reward": 0.7291666865348816,383 "reward_std": 0.30859363824129105,384 "rewards/correct_code_reward_func": 0.2708333432674408,385 "rewards/len_reward_func": 0.4583333432674408,386 "step": 29387 },388 {389 "completion_length": 91.70833587646484,390 "epoch": 0.39473684210526316,391 "grad_norm": 1.2731739041206802,392 "kl": 0.03851318359375,393 "learning_rate": 4.867563208197519e-07,394 "loss": 0.0,395 "reward": 0.799913227558136,396 "reward_std": 0.3069130629301071,397 "rewards/correct_code_reward_func": 0.3541666865348816,398 "rewards/len_reward_func": 0.4457465559244156,399 "step": 30400 },401 {402 "completion_length": 72.29166793823242,403 "epoch": 0.40789473684210525,404 "grad_norm": 1.4676508584854064,405 "kl": 0.04327392578125,406 "learning_rate": 4.855910911906906e-07,407 "loss": 0.0,408 "reward": 0.9658634960651398,409 "reward_std": 0.3308482989668846,410 "rewards/correct_code_reward_func": 0.520833358168602,411 "rewards/len_reward_func": 0.4450301378965378,412 "step": 31413 },414 {415 "completion_length": 76.93750381469727,416 "epoch": 0.42105263157894735,417 "grad_norm": 4.605001948885057,418 "kl": 0.087890625,419 "learning_rate": 4.843782550073969e-07,420 "loss": 0.0001,421 "reward": 0.8541666865348816,422 "reward_std": 0.22516431659460068,423 "rewards/correct_code_reward_func": 0.375,424 "rewards/len_reward_func": 0.4791666716337204,425 "step": 32426 },427 {428 "completion_length": 55.77083396911621,429 "epoch": 0.4342105263157895,430 "grad_norm": 1.0747031450058906,431 "kl": 0.234375,432 "learning_rate": 4.83118057351089e-07,433 "loss": 0.0002,434 "reward": 0.6458333432674408,435 "reward_std": 0.16340987384319305,436 "rewards/correct_code_reward_func": 0.1666666716337204,437 "rewards/len_reward_func": 0.4791666716337204,438 "step": 33439 },440 {441 "completion_length": 58.02083396911621,442 "epoch": 0.4473684210526316,443 "grad_norm": 3.450736535043091,444 "kl": 0.06573486328125,445 "learning_rate": 4.818107528734503e-07,446 "loss": 0.0001,447 "reward": 0.9100490808486938,448 "reward_std": 0.49446116387844086,449 "rewards/correct_code_reward_func": 0.4375,450 "rewards/len_reward_func": 0.47254903614521027,451 "step": 34452 },453 {454 "completion_length": 64.47916793823242,455 "epoch": 0.4605263157894737,456 "grad_norm": 3.8860905534674637,457 "kl": 0.0714111328125,458 "learning_rate": 4.804566057451729e-07,459 "loss": 0.0001,460 "reward": 0.8531287312507629,461 "reward_std": 0.3388620540499687,462 "rewards/correct_code_reward_func": 0.3541666865348816,463 "rewards/len_reward_func": 0.49896204471588135,464 "step": 35465 },466 {467 "completion_length": 49.06250286102295,468 "epoch": 0.47368421052631576,469 "grad_norm": 118.2111884355624,470 "kl": 0.9837646484375,471 "learning_rate": 4.790558896025742e-07,472 "loss": 0.001,473 "reward": 0.8689024448394775,474 "reward_std": 0.31673362106084824,475 "rewards/correct_code_reward_func": 0.375,476 "rewards/len_reward_func": 0.49390244483947754,477 "step": 36478 },479 {480 "completion_length": 82.68750381469727,481 "epoch": 0.4868421052631579,482 "grad_norm": 1.2725798743612686,483 "kl": 0.0997314453125,484 "learning_rate": 4.776088874923041e-07,485 "loss": 0.0001,486 "reward": 0.7270315587520599,487 "reward_std": 0.36151498556137085,488 "rewards/correct_code_reward_func": 0.229166679084301,489 "rewards/len_reward_func": 0.49786490201950073,490 "step": 37491 },492 {493 "completion_length": 67.29166984558105,494 "epoch": 0.5,495 "grad_norm": 0.6682530578144538,496 "kl": 0.24072265625,497 "learning_rate": 4.7611589181414734e-07,498 "loss": 0.0002,499 "reward": 0.7083333432674408,500 "reward_std": 0.19500282034277916,501 "rewards/correct_code_reward_func": 0.2083333432674408,502 "rewards/len_reward_func": 0.5,503 "step": 38504 },505 {506 "completion_length": 82.89583587646484,507 "epoch": 0.5131578947368421,508 "grad_norm": 1.896117750609313,509 "kl": 0.109375,510 "learning_rate": 4.745772042619388e-07,511 "loss": 0.0001,512 "reward": 0.7291666865348816,513 "reward_std": 0.28126102685928345,514 "rewards/correct_code_reward_func": 0.229166679084301,515 "rewards/len_reward_func": 0.5,516 "step": 39517 },518 {519 "completion_length": 58.06250190734863,520 "epoch": 0.5263157894736842,521 "grad_norm": 118.0980466511993,522 "kl": 44.5732421875,523 "learning_rate": 4.729931357625986e-07,524 "loss": 0.0447,525 "reward": 0.7083333432674408,526 "reward_std": 0.3177001550793648,527 "rewards/correct_code_reward_func": 0.2083333432674408,528 "rewards/len_reward_func": 0.5,529 "step": 40530 },531 {532 "completion_length": 74.12500381469727,533 "epoch": 0.5394736842105263,534 "grad_norm": 0.7236725923896844,535 "kl": 0.41094970703125,536 "learning_rate": 4.7136400641330245e-07,537 "loss": 0.0004,538 "reward": 0.6250000149011612,539 "reward_std": 0.1178511306643486,540 "rewards/correct_code_reward_func": 0.1458333432674408,541 "rewards/len_reward_func": 0.4791666716337204,542 "step": 41543 },544 {545 "completion_length": 46.895835876464844,546 "epoch": 0.5526315789473685,547 "grad_norm": 4.631491683010619,548 "kl": 1.0185546875,549 "learning_rate": 4.696901454167988e-07,550 "loss": 0.001,551 "reward": 0.9166666865348816,552 "reward_std": 0.2903675064444542,553 "rewards/correct_code_reward_func": 0.4166666865348816,554 "rewards/len_reward_func": 0.5,555 "step": 42556 },557 {558 "completion_length": 60.79166793823242,559 "epoch": 0.5657894736842105,560 "grad_norm": 3.3754705022635965,561 "kl": 0.15283203125,562 "learning_rate": 4.6797189101488576e-07,563 "loss": 0.0002,564 "reward": 0.979166716337204,565 "reward_std": 0.1767766959965229,566 "rewards/correct_code_reward_func": 0.5000000149011612,567 "rewards/len_reward_func": 0.4791666716337204,568 "step": 43569 },570 {571 "completion_length": 43.583335876464844,572 "epoch": 0.5789473684210527,573 "grad_norm": 95.77831861506337,574 "kl": 2.21484375,575 "learning_rate": 4.662095904200617e-07,576 "loss": 0.0022,577 "reward": 0.9375000596046448,578 "reward_std": 0.3310800567269325,579 "rewards/correct_code_reward_func": 0.4375000111758709,580 "rewards/len_reward_func": 0.5,581 "step": 44582 },583 {584 "completion_length": 64.0625,585 "epoch": 0.5921052631578947,586 "grad_norm": 0.9899941998859535,587 "kl": 0.0555419921875,588 "learning_rate": 4.6440359974536304e-07,589 "loss": 0.0001,590 "reward": 0.7708333730697632,591 "reward_std": 0.16340987384319305,592 "rewards/correct_code_reward_func": 0.2708333432674408,593 "rewards/len_reward_func": 0.5,594 "step": 45595 },596 {597 "completion_length": 53.04166793823242,598 "epoch": 0.6052631578947368,599 "grad_norm": 3.552556057212125,600 "kl": 0.229736328125,601 "learning_rate": 4.6255428393240354e-07,602 "loss": 0.0002,603 "reward": 1.0833333730697632,604 "reward_std": 0.19500282034277916,605 "rewards/correct_code_reward_func": 0.5833333432674408,606 "rewards/len_reward_func": 0.5,607 "step": 46608 },609 {610 "completion_length": 40.52083396911621,611 "epoch": 0.618421052631579,612 "grad_norm": 27.248705920564117,613 "kl": 7.90234375,614 "learning_rate": 4.606620166776294e-07,615 "loss": 0.0079,616 "reward": 0.8750000298023224,617 "reward_std": 0.16623876243829727,618 "rewards/correct_code_reward_func": 0.3750000223517418,619 "rewards/len_reward_func": 0.5,620 "step": 47621 },622 {623 "completion_length": 54.68750190734863,624 "epoch": 0.631578947368421,625 "grad_norm": 6.398561296574553,626 "kl": 0.744384765625,627 "learning_rate": 4.587271803568055e-07,628 "loss": 0.0007,629 "reward": 0.5416666865348816,630 "reward_std": 0.1451837718486786,631 "rewards/correct_code_reward_func": 0.0625,632 "rewards/len_reward_func": 0.4791666716337204,633 "step": 48634 },635 {636 "completion_length": 48.29166793823242,637 "epoch": 0.6447368421052632,638 "grad_norm": 4.1122533170941535,639 "kl": 0.148681640625,640 "learning_rate": 4.567501659477476e-07,641 "loss": 0.0001,642 "reward": 0.8958333730697632,643 "reward_std": 0.3205290287733078,644 "rewards/correct_code_reward_func": 0.3958333432674408,645 "rewards/len_reward_func": 0.5,646 "step": 49647 },648 {649 "completion_length": 39.3125,650 "epoch": 0.6578947368421053,651 "grad_norm": 53.38113189457624,652 "kl": 13.2625732421875,653 "learning_rate": 4.5473137295131625e-07,654 "loss": 0.0133,655 "reward": 0.9375,656 "reward_std": 0.22516433894634247,657 "rewards/correct_code_reward_func": 0.4375,658 "rewards/len_reward_func": 0.5,659 "step": 50660 },661 {662 "completion_length": 42.83333396911621,663 "epoch": 0.6710526315789473,664 "grad_norm": 18178.959749921032,665 "kl": 4768.09375,666 "learning_rate": 4.526712093106887e-07,667 "loss": 4.7525,668 "reward": 1.0208333432674408,669 "reward_std": 0.2041093371808529,670 "rewards/correct_code_reward_func": 0.5208333432674408,671 "rewards/len_reward_func": 0.5,672 "step": 51673 },674 {675 "completion_length": 71.66667175292969,676 "epoch": 0.6842105263157895,677 "grad_norm": 1.9579852812558598,678 "kl": 0.474609375,679 "learning_rate": 4.5057009132892455e-07,680 "loss": 0.0005,681 "reward": 0.7916666865348816,682 "reward_std": 0.1178511306643486,683 "rewards/correct_code_reward_func": 0.3333333432674408,684 "rewards/len_reward_func": 0.4583333432674408,685 "step": 52686 },687 {688 "completion_length": 57.83333396911621,689 "epoch": 0.6973684210526315,690 "grad_norm": 3.4888745953712426,691 "kl": 0.9111328125,692 "learning_rate": 4.4842844358484233e-07,693 "loss": 0.0009,694 "reward": 0.9375000298023224,695 "reward_std": 0.22516431659460068,696 "rewards/correct_code_reward_func": 0.4375000149011612,697 "rewards/len_reward_func": 0.5,698 "step": 53699 },700 {701 "completion_length": 86.72916793823242,702 "epoch": 0.7105263157894737,703 "grad_norm": 3.0697908089632087,704 "kl": 0.12158203125,705 "learning_rate": 4.4624669884722364e-07,706 "loss": 0.0001,707 "reward": 0.8333333432674408,708 "reward_std": 0.31142252683639526,709 "rewards/correct_code_reward_func": 0.3333333432674408,710 "rewards/len_reward_func": 0.5,711 "step": 54712 },713 {714 "completion_length": 53.895835876464844,715 "epoch": 0.7236842105263158,716 "grad_norm": 1.4628769742470222,717 "kl": 0.137939453125,718 "learning_rate": 4.4402529798736214e-07,719 "loss": 0.0001,720 "reward": 1.0,721 "reward_std": 0.2994871214032173,722 "rewards/correct_code_reward_func": 0.5,723 "rewards/len_reward_func": 0.5,724 "step": 55725 },726 {727 "completion_length": 51.770835876464844,728 "epoch": 0.7368421052631579,729 "grad_norm": 19.218612806133308,730 "kl": 6.68017578125,731 "learning_rate": 4.417646898899758e-07,732 "loss": 0.0067,733 "reward": 0.7708333432674408,734 "reward_std": 0.2041093371808529,735 "rewards/correct_code_reward_func": 0.2708333358168602,736 "rewards/len_reward_func": 0.5,737 "step": 56738 },739 {740 "completion_length": 48.875,741 "epoch": 0.75,742 "grad_norm": 1.1426530146279454,743 "kl": 0.115234375,744 "learning_rate": 4.394653313624992e-07,745 "loss": 0.0001,746 "reward": 0.9791666865348816,747 "reward_std": 0.3478616699576378,748 "rewards/correct_code_reward_func": 0.5000000149011612,749 "rewards/len_reward_func": 0.4791666716337204,750 "step": 57751 },752 {753 "completion_length": 39.91666793823242,754 "epoch": 0.7631578947368421,755 "grad_norm": 1.424776813399897,756 "kl": 2.8671875,757 "learning_rate": 4.3712768704277524e-07,758 "loss": 0.0029,759 "reward": 0.8333333432674408,760 "reward_std": 0.22233545035123825,761 "rewards/correct_code_reward_func": 0.3333333432674408,762 "rewards/len_reward_func": 0.5,763 "step": 58764 },765 {766 "completion_length": 70.31250190734863,767 "epoch": 0.7763157894736842,768 "grad_norm": 3.1751573512407827,769 "kl": 0.115234375,770 "learning_rate": 4.3475222930516473e-07,771 "loss": 0.0001,772 "reward": 0.873985230922699,773 "reward_std": 0.25253836810588837,774 "rewards/correct_code_reward_func": 0.3958333432674408,775 "rewards/len_reward_func": 0.4781518578529358,776 "step": 59777 },778 {779 "completion_length": 54.54166793823242,780 "epoch": 0.7894736842105263,781 "grad_norm": 58.36302253834897,782 "kl": 1.03515625,783 "learning_rate": 4.32339438165092e-07,784 "loss": 0.001,785 "reward": 0.916666716337204,786 "reward_std": 0.42927367985248566,787 "rewards/correct_code_reward_func": 0.416666679084301,788 "rewards/len_reward_func": 0.5,789 "step": 60790 },791 {792 "completion_length": 63.583335876464844,793 "epoch": 0.8026315789473685,794 "grad_norm": 2.827911135736344,795 "kl": 0.29736328125,796 "learning_rate": 4.29889801182047e-07,797 "loss": 0.0003,798 "reward": 0.791666716337204,799 "reward_std": 0.1451837606728077,800 "rewards/correct_code_reward_func": 0.29166667722165585,801 "rewards/len_reward_func": 0.5,802 "step": 61803 },804 {805 "completion_length": 65.41666793823242,806 "epoch": 0.8157894736842105,807 "grad_norm": 1.5958266464282538,808 "kl": 0.065673828125,809 "learning_rate": 4.274038133610628e-07,810 "loss": 0.0001,811 "reward": 0.8333333730697632,812 "reward_std": 0.15430335700511932,813 "rewards/correct_code_reward_func": 0.3333333432674408,814 "rewards/len_reward_func": 0.5,815 "step": 62816 },817 {818 "completion_length": 31.270834922790527,819 "epoch": 0.8289473684210527,820 "grad_norm": 6.489845534414021,821 "kl": 1.37109375,822 "learning_rate": 4.248819770526884e-07,823 "loss": 0.0014,824 "reward": 1.1250000298023224,825 "reward_std": 0.376638799905777,826 "rewards/correct_code_reward_func": 0.6250000298023224,827 "rewards/len_reward_func": 0.5,828 "step": 63829 },830 {831 "completion_length": 27.750000953674316,832 "epoch": 0.8421052631578947,833 "grad_norm": 1.7297180083254418,834 "kl": 4.314453125,835 "learning_rate": 4.223248018514777e-07,836 "loss": 0.0043,837 "reward": 1.1458333730697632,838 "reward_std": 0.30859364569187164,839 "rewards/correct_code_reward_func": 0.6458333432674408,840 "rewards/len_reward_func": 0.5,841 "step": 64842 },843 {844 "completion_length": 47.35416793823242,845 "epoch": 0.8552631578947368,846 "grad_norm": 15.360105674599206,847 "kl": 0.5107421875,848 "learning_rate": 4.1973280449301364e-07,849 "loss": 0.0005,850 "reward": 1.0833333730697632,851 "reward_std": 0.3177001550793648,852 "rewards/correct_code_reward_func": 0.5833333432674408,853 "rewards/len_reward_func": 0.5,854 "step": 65855 },856 {857 "completion_length": 34.500000953674316,858 "epoch": 0.868421052631579,859 "grad_norm": 0.8012424945081584,860 "kl": 0.1363525390625,861 "learning_rate": 4.171065087494909e-07,862 "loss": 0.0001,863 "reward": 0.7083333730697632,864 "reward_std": 0.07715167850255966,865 "rewards/correct_code_reward_func": 0.2083333395421505,866 "rewards/len_reward_func": 0.5,867 "step": 66868 },869 {870 "completion_length": 68.00000190734863,871 "epoch": 0.881578947368421,872 "grad_norm": 1.8876444286085294,873 "kl": 0.2359619140625,874 "learning_rate": 4.144464453238748e-07,875 "loss": 0.0002,876 "reward": 0.9583333730697632,877 "reward_std": 0.36751919239759445,878 "rewards/correct_code_reward_func": 0.4791666716337204,879 "rewards/len_reward_func": 0.4791666716337204,880 "step": 67881 },882 {883 "completion_length": 111.93750762939453,884 "epoch": 0.8947368421052632,885 "grad_norm": 0.7306458054174331,886 "kl": 0.102294921875,887 "learning_rate": 4.1175315174266135e-07,888 "loss": 0.0001,889 "reward": 0.5625000298023224,890 "reward_std": 0.1767766959965229,891 "rewards/correct_code_reward_func": 0.06250000186264515,892 "rewards/len_reward_func": 0.5,893 "step": 68894 },895 {896 "completion_length": 92.27083969116211,897 "epoch": 0.9078947368421053,898 "grad_norm": 1.8168097569883273,899 "kl": 0.185546875,900 "learning_rate": 4.090271722472576e-07,901 "loss": 0.0002,902 "reward": 0.9791666865348816,903 "reward_std": 0.28126100450754166,904 "rewards/correct_code_reward_func": 0.5,905 "rewards/len_reward_func": 0.4791666716337204,906 "step": 69907 },908 {909 "completion_length": 86.77083587646484,910 "epoch": 0.9210526315789473,911 "grad_norm": 2.3675233810359515,912 "kl": 1.0478515625,913 "learning_rate": 4.062690576840051e-07,914 "loss": 0.001,915 "reward": 0.8541666865348816,916 "reward_std": 0.32964862883090973,917 "rewards/correct_code_reward_func": 0.3541666865348816,918 "rewards/len_reward_func": 0.5,919 "step": 70920 },921 {922 "completion_length": 57.66666793823242,923 "epoch": 0.9342105263157895,924 "grad_norm": 51.022759357987056,925 "kl": 8.755859375,926 "learning_rate": 4.0347936539286874e-07,927 "loss": 0.0088,928 "reward": 1.0,929 "reward_std": 0.19500282034277916,930 "rewards/correct_code_reward_func": 0.5,931 "rewards/len_reward_func": 0.5,932 "step": 71933 },934 {935 "completion_length": 79.37500381469727,936 "epoch": 0.9473684210526315,937 "grad_norm": 1.1663678155323538,938 "kl": 0.1044921875,939 "learning_rate": 4.006586590948141e-07,940 "loss": 0.0001,941 "reward": 1.048139214515686,942 "reward_std": 0.17669584602117538,943 "rewards/correct_code_reward_func": 0.5625000149011612,944 "rewards/len_reward_func": 0.48563915491104126,945 "step": 72946 },947 {948 "completion_length": 35.62500190734863,949 "epoch": 0.9605263157894737,950 "grad_norm": 1.1660255366197148,951 "kl": 0.2127685546875,952 "learning_rate": 3.9780750877789394e-07,953 "loss": 0.0002,954 "reward": 0.9375000596046448,955 "reward_std": 0.24056154489517212,956 "rewards/correct_code_reward_func": 0.4375000149011612,957 "rewards/len_reward_func": 0.5,958 "step": 73959 },960 {961 "completion_length": 63.70833396911621,962 "epoch": 0.9736842105263158,963 "grad_norm": 1.1033519234822637,964 "kl": 0.115234375,965 "learning_rate": 3.9492649058206964e-07,966 "loss": 0.0001,967 "reward": 0.8333333730697632,968 "reward_std": 0.3491683229804039,969 "rewards/correct_code_reward_func": 0.3541666679084301,970 "rewards/len_reward_func": 0.4791666716337204,971 "step": 74972 },973 {974 "completion_length": 76.02083396911621,975 "epoch": 0.9868421052631579,976 "grad_norm": 1.0147417730649864,977 "kl": 0.085693359375,978 "learning_rate": 3.920161866827889e-07,979 "loss": 0.0001,980 "reward": 0.9223043918609619,981 "reward_std": 0.26814398169517517,982 "rewards/correct_code_reward_func": 0.4375,983 "rewards/len_reward_func": 0.4848043918609619,984 "step": 75985 },986 {987 "completion_length": 37.04166793823242,988 "epoch": 1.0,989 "grad_norm": 53.40321846717754,990 "kl": 1.2109375,991 "learning_rate": 3.8907718517334396e-07,992 "loss": 0.0012,993 "reward": 1.125,994 "reward_std": 0.2994871288537979,995 "rewards/correct_code_reward_func": 0.625,996 "rewards/len_reward_func": 0.5,997 "step": 76998 },999 {1000 "completion_length": 109.77083587646484,1001 "epoch": 1.013157894736842,1002 "grad_norm": 1.2927063784991473,1003 "kl": 0.1328125,1004 "learning_rate": 3.861100799460336e-07,1005 "loss": 0.0001,1006 "reward": 0.6458333730697632,1007 "reward_std": 0.16340987384319305,1008 "rewards/correct_code_reward_func": 0.1458333395421505,1009 "rewards/len_reward_func": 0.5,1010 "step": 771011 },1012 {1013 "completion_length": 55.6875,1014 "epoch": 1.0263157894736843,1015 "grad_norm": 1.118771460054152,1016 "kl": 0.130859375,1017 "learning_rate": 3.831154705721541e-07,1018 "loss": 0.0001,1019 "reward": 1.0,1020 "reward_std": 0.2342708371579647,1021 "rewards/correct_code_reward_func": 0.5,1022 "rewards/len_reward_func": 0.5,1023 "step": 781024 },1025 {1026 "completion_length": 46.25000190734863,1027 "epoch": 1.0394736842105263,1028 "grad_norm": 6.523486555479139,1029 "kl": 0.462158203125,1030 "learning_rate": 3.800939621808419e-07,1031 "loss": 0.0005,1032 "reward": 0.9583333432674408,1033 "reward_std": 0.22233543917536736,1034 "rewards/correct_code_reward_func": 0.4583333432674408,1035 "rewards/len_reward_func": 0.5,1036 "step": 791037 },1038 {1039 "completion_length": 77.52083587646484,1040 "epoch": 1.0526315789473684,1041 "grad_norm": 1.2611203201996803,1042 "kl": 0.10211181640625,1043 "learning_rate": 3.7704616533679334e-07,1044 "loss": 0.0001,1045 "reward": 0.7708333432674408,1046 "reward_std": 0.438380166888237,1047 "rewards/correct_code_reward_func": 0.2708333432674408,1048 "rewards/len_reward_func": 0.5,1049 "step": 801050 },1051 {1052 "completion_length": 31.666666984558105,1053 "epoch": 1.0657894736842106,1054 "grad_norm": 6780.981931743591,1055 "kl": 2705.0,1056 "learning_rate": 3.7397269591688663e-07,1057 "loss": 2.7153,1058 "reward": 0.9375,1059 "reward_std": 0.13607724383473396,1060 "rewards/correct_code_reward_func": 0.4375,1061 "rewards/len_reward_func": 0.5,1062 "step": 811063 },1064 {1065 "completion_length": 60.08333396911621,1066 "epoch": 1.0789473684210527,1067 "grad_norm": 1.4225091724950767,1068 "kl": 0.09033203125,1069 "learning_rate": 3.7087417498572944e-07,1070 "loss": 0.0001,1071 "reward": 0.7916666865348816,1072 "reward_std": 0.34018656611442566,1073 "rewards/correct_code_reward_func": 0.2916666716337204,1074 "rewards/len_reward_func": 0.5,1075 "step": 821076 },1077 {1078 "completion_length": 47.95833396911621,1079 "epoch": 1.0921052631578947,1080 "grad_norm": 39.92181658742887,1081 "kl": 21.05078125,1082 "learning_rate": 3.6775122867015865e-07,1083 "loss": 0.021,1084 "reward": 1.1666667461395264,1085 "reward_std": 0.34503278136253357,1086 "rewards/correct_code_reward_func": 0.6666666865348816,1087 "rewards/len_reward_func": 0.5,1088 "step": 831089 },1090 {1091 "completion_length": 30.291667938232422,1092 "epoch": 1.1052631578947367,1093 "grad_norm": 1.8228407124659103,1094 "kl": 0.380859375,1095 "learning_rate": 3.6460448803271754e-07,1096 "loss": 0.0004,1097 "reward": 1.1875000596046448,1098 "reward_std": 0.2931964099407196,1099 "rewards/correct_code_reward_func": 0.6875000298023224,1100 "rewards/len_reward_func": 0.5,1101 "step": 841102 },1103 {1104 "completion_length": 48.1875,1105 "epoch": 1.118421052631579,1106 "grad_norm": 0.7759500560241989,1107 "kl": 0.359130859375,1108 "learning_rate": 3.614345889441346e-07,1109 "loss": 0.0004,1110 "reward": 0.7019230872392654,1111 "reward_std": 0.09528262168169022,1112 "rewards/correct_code_reward_func": 0.2083333432674408,1113 "rewards/len_reward_func": 0.49358974397182465,1114 "step": 851115 },1116 {1117 "completion_length": 31.58333396911621,1118 "epoch": 1.131578947368421,1119 "grad_norm": 522.7574671201318,1120 "kl": 133.271484375,1121 "learning_rate": 3.5824217195483176e-07,1122 "loss": 0.1335,1123 "reward": 0.8541666865348816,1124 "reward_std": 0.13607724383473396,1125 "rewards/correct_code_reward_func": 0.3541666679084301,1126 "rewards/len_reward_func": 0.5,1127 "step": 861128 },1129 {1130 "completion_length": 41.29166793823242,1131 "epoch": 1.1447368421052633,1132 "grad_norm": 1.2754536374690895,1133 "kl": 0.64208984375,1134 "learning_rate": 3.550278821654866e-07,1135 "loss": 0.0006,1136 "reward": 1.0208333730697632,1137 "reward_std": 0.16340987384319305,1138 "rewards/correct_code_reward_func": 0.520833358168602,1139 "rewards/len_reward_func": 0.5,1140 "step": 871141 },1142 {1143 "completion_length": 49.52083396911621,1144 "epoch": 1.1578947368421053,1145 "grad_norm": 1.1388596254339887,1146 "kl": 0.1265869140625,1147 "learning_rate": 3.5179236909667464e-07,1148 "loss": 0.0001,1149 "reward": 1.0,1150 "reward_std": 0.1178511306643486,1151 "rewards/correct_code_reward_func": 0.5,1152 "rewards/len_reward_func": 0.5,1153 "step": 881154 },1155 {1156 "completion_length": 56.104169845581055,1157 "epoch": 1.1710526315789473,1158 "grad_norm": 4.8354182485526875,1159 "kl": 1.86669921875,1160 "learning_rate": 3.485362865576194e-07,1161 "loss": 0.0019,1162 "reward": 0.8541666865348816,1163 "reward_std": 0.16340987384319305,1164 "rewards/correct_code_reward_func": 0.3541666865348816,1165 "rewards/len_reward_func": 0.5,1166 "step": 891167 },1168 {1169 "completion_length": 55.562503814697266,1170 "epoch": 1.1842105263157894,1171 "grad_norm": 1.8509215827690364,1172 "kl": 0.501953125,1173 "learning_rate": 3.4526029251407505e-07,1174 "loss": 0.0005,1175 "reward": 0.6250000298023224,1176 "reward_std": 0.22233543917536736,1177 "rewards/correct_code_reward_func": 0.12500000558793545,1178 "rewards/len_reward_func": 0.5,1179 "step": 901180 },1181 {1182 "completion_length": 56.89583396911621,1183 "epoch": 1.1973684210526316,1184 "grad_norm": 3.479508392096908,1185 "kl": 0.2423095703125,1186 "learning_rate": 3.4196504895536943e-07,1187 "loss": 0.0002,1188 "reward": 0.9166666865348816,1189 "reward_std": 0.24383928626775742,1190 "rewards/correct_code_reward_func": 0.4375000149011612,1191 "rewards/len_reward_func": 0.4791666716337204,1192 "step": 911193 },1194 {1195 "completion_length": 49.79166793823242,1196 "epoch": 1.2105263157894737,1197 "grad_norm": 1.436722605439725,1198 "kl": 0.130126953125,1199 "learning_rate": 3.3865122176063385e-07,1200 "loss": 0.0001,