CPunisher/JavaBench
1
1{2 "completion": [3 {4 "Context": "selective",5 "Method": "holistic",6 "Model": "gpt-4o-2024-05-13",7 "Pass_at_1": 1.08 },9 {10 "Context": "selective",11 "Method": "holistic",12 "Model": "deepseek-coder-6.7b-instruct",13 "Pass_at_1": 0.937857142914 },15 {16 "Context": "selective",17 "Method": "holistic",18 "Model": "deepseek-coder-33b-instruct",19 "Pass_at_1": 0.935714285720 },21 {22 "Context": "selective",23 "Method": "holistic",24 "Model": "gpt-3.5-turbo-1106",25 "Pass_at_1": 0.921428571426 },27 {28 "Context": "selective",29 "Method": "incremental",30 "Model": "deepseek-coder-6.7b-instruct",31 "Pass_at_1": 0.864285714332 },33 {34 "Context": "selective",35 "Method": "independent",36 "Model": "gpt-3.5-turbo-1106",37 "Pass_at_1": 0.853571428638 },39 {40 "Context": "selective",41 "Method": "independent",42 "Model": "deepseek-coder-6.7b-instruct",43 "Pass_at_1": 0.847142857144 },45 {46 "Context": "selective",47 "Method": "holistic",48 "Model": "Phind-CodeLlama-34B-v2",49 "Pass_at_1": 0.84550 },51 {52 "Context": "selective",53 "Method": "incremental",54 "Model": "gpt-3.5-turbo-1106",55 "Pass_at_1": 0.839285714356 },57 {58 "Context": "selective",59 "Method": "independent",60 "Model": "deepseek-coder-33b-instruct",61 "Pass_at_1": 0.819285714362 },63 {64 "Context": "selective",65 "Method": "holistic",66 "Model": "WizardCoder-15B-V1.0",67 "Pass_at_1": 0.796428571468 },69 {70 "Context": "selective",71 "Method": "incremental",72 "Model": "deepseek-coder-33b-instruct",73 "Pass_at_1": 0.748571428674 },75 {76 "Context": "selective",77 "Method": "independent",78 "Model": "Phind-CodeLlama-34B-v2",79 "Pass_at_1": 0.637857142980 },81 {82 "Context": "selective",83 "Method": "incremental",84 "Model": "Phind-CodeLlama-34B-v2",85 "Pass_at_1": 0.599285714386 },87 {88 "Context": "selective",89 "Method": "independent",90 "Model": "WizardCoder-15B-V1.0",91 "Pass_at_1": 0.597142857192 },93 {94 "Context": "selective",95 "Method": "incremental",96 "Model": "WizardCoder-15B-V1.0",97 "Pass_at_1": 0.505714285798 }99 ],100 "compilation_class_wise": [101 {102 "Context": "selective",103 "Method": "holistic",104 "Model": "gpt-3.5-turbo-1106",105 "Pass_at_1": 0.7942857143106 },107 {108 "Context": "selective",109 "Method": "holistic",110 "Model": "deepseek-coder-33b-instruct",111 "Pass_at_1": 0.7414285714112 },113 {114 "Context": "selective",115 "Method": "incremental",116 "Model": "deepseek-coder-6.7b-instruct",117 "Pass_at_1": 0.725118 },119 {120 "Context": "selective",121 "Method": "holistic",122 "Model": "deepseek-coder-6.7b-instruct",123 "Pass_at_1": 0.7171428571124 },125 {126 "Context": "selective",127 "Method": "incremental",128 "Model": "gpt-3.5-turbo-1106",129 "Pass_at_1": 0.7130 },131 {132 "Context": "selective",133 "Method": "holistic",134 "Model": "Phind-CodeLlama-34B-v2",135 "Pass_at_1": 0.6978571429136 },137 {138 "Context": "selective",139 "Method": "independent",140 "Model": "gpt-3.5-turbo-1106",141 "Pass_at_1": 0.6828571429142 },143 {144 "Context": "selective",145 "Method": "independent",146 "Model": "deepseek-coder-6.7b-instruct",147 "Pass_at_1": 0.6814285714148 },149 {150 "Context": "selective",151 "Method": "independent",152 "Model": "deepseek-coder-33b-instruct",153 "Pass_at_1": 0.6764285714154 },155 {156 "Context": "selective",157 "Method": "holistic",158 "Model": "gpt-4o-2024-05-13",159 "Pass_at_1": 0.6607142857160 },161 {162 "Context": "selective",163 "Method": "holistic",164 "Model": "WizardCoder-15B-V1.0",165 "Pass_at_1": 0.58166 },167 {168 "Context": "selective",169 "Method": "incremental",170 "Model": "deepseek-coder-33b-instruct",171 "Pass_at_1": 0.5778571429172 },173 {174 "Context": "selective",175 "Method": "incremental",176 "Model": "Phind-CodeLlama-34B-v2",177 "Pass_at_1": 0.5021428571178 },179 {180 "Context": "selective",181 "Method": "independent",182 "Model": "Phind-CodeLlama-34B-v2",183 "Pass_at_1": 0.4985714286184 },185 {186 "Context": "selective",187 "Method": "independent",188 "Model": "WizardCoder-15B-V1.0",189 "Pass_at_1": 0.4335714286190 },191 {192 "Context": "selective",193 "Method": "incremental",194 "Model": "WizardCoder-15B-V1.0",195 "Pass_at_1": 0.3507142857196 }197 ],198 "compilation_test_wise": [199 {200 "Context": "selective",201 "Method": "holistic",202 "Model": "gpt-4o-2024-05-13",203 "Pass_at_1": 0.5035714286204 },205 {206 "Context": "selective",207 "Method": "holistic",208 "Model": "deepseek-coder-33b-instruct",209 "Pass_at_1": 0.4202826585210 },211 {212 "Context": "selective",213 "Method": "holistic",214 "Model": "gpt-3.5-turbo-1106",215 "Pass_at_1": 0.3443277311216 },217 {218 "Context": "selective",219 "Method": "independent",220 "Model": "deepseek-coder-33b-instruct",221 "Pass_at_1": 0.3378676471222 },223 {224 "Context": "selective",225 "Method": "holistic",226 "Model": "Phind-CodeLlama-34B-v2",227 "Pass_at_1": 0.3183823529228 },229 {230 "Context": "selective",231 "Method": "holistic",232 "Model": "deepseek-coder-6.7b-instruct",233 "Pass_at_1": 0.3121848739234 },235 {236 "Context": "selective",237 "Method": "independent",238 "Model": "deepseek-coder-6.7b-instruct",239 "Pass_at_1": 0.2836134454240 },241 {242 "Context": "selective",243 "Method": "incremental",244 "Model": "deepseek-coder-6.7b-instruct",245 "Pass_at_1": 0.2621848739246 },247 {248 "Context": "selective",249 "Method": "holistic",250 "Model": "WizardCoder-15B-V1.0",251 "Pass_at_1": 0.2240546218252 },253 {254 "Context": "selective",255 "Method": "independent",256 "Model": "gpt-3.5-turbo-1106",257 "Pass_at_1": 0.1930147059258 },259 {260 "Context": "selective",261 "Method": "incremental",262 "Model": "gpt-3.5-turbo-1106",263 "Pass_at_1": 0.0955357143264 },265 {266 "Context": "selective",267 "Method": "independent",268 "Model": "Phind-CodeLlama-34B-v2",269 "Pass_at_1": 0.0932773109270 },271 {272 "Context": "selective",273 "Method": "incremental",274 "Model": "Phind-CodeLlama-34B-v2",275 "Pass_at_1": 0.0701680672276 },277 {278 "Context": "selective",279 "Method": "incremental",280 "Model": "deepseek-coder-33b-instruct",281 "Pass_at_1": 0.0352941176282 },283 {284 "Context": "selective",285 "Method": "independent",286 "Model": "WizardCoder-15B-V1.0",287 "Pass_at_1": 0.0288 },289 {290 "Context": "selective",291 "Method": "incremental",292 "Model": "WizardCoder-15B-V1.0",293 "Pass_at_1": 0.0294 }295 ],296 "pass_class_wise": [297 {298 "Context": "selective",299 "Method": "holistic",300 "Model": "gpt-3.5-turbo-1106",301 "Pass_at_1": 0.7832360347302 },303 {304 "Context": "selective",305 "Method": "holistic",306 "Model": "deepseek-coder-33b-instruct",307 "Pass_at_1": 0.723699056308 },309 {310 "Context": "selective",311 "Method": "incremental",312 "Model": "deepseek-coder-6.7b-instruct",313 "Pass_at_1": 0.7036481325314 },315 {316 "Context": "selective",317 "Method": "holistic",318 "Model": "deepseek-coder-6.7b-instruct",319 "Pass_at_1": 0.6855203826320 },321 {322 "Context": "selective",323 "Method": "incremental",324 "Model": "gpt-3.5-turbo-1106",325 "Pass_at_1": 0.6846699639326 },327 {328 "Context": "selective",329 "Method": "holistic",330 "Model": "Phind-CodeLlama-34B-v2",331 "Pass_at_1": 0.6808480861332 },333 {334 "Context": "selective",335 "Method": "independent",336 "Model": "gpt-3.5-turbo-1106",337 "Pass_at_1": 0.6772858762338 },339 {340 "Context": "selective",341 "Method": "independent",342 "Model": "deepseek-coder-6.7b-instruct",343 "Pass_at_1": 0.6547244155344 },345 {346 "Context": "selective",347 "Method": "independent",348 "Model": "deepseek-coder-33b-instruct",349 "Pass_at_1": 0.6547232007350 },351 {352 "Context": "selective",353 "Method": "holistic",354 "Model": "gpt-4o-2024-05-13",355 "Pass_at_1": 0.6545897285356 },357 {358 "Context": "selective",359 "Method": "holistic",360 "Model": "WizardCoder-15B-V1.0",361 "Pass_at_1": 0.5674101922362 },363 {364 "Context": "selective",365 "Method": "incremental",366 "Model": "deepseek-coder-33b-instruct",367 "Pass_at_1": 0.5603969625368 },369 {370 "Context": "selective",371 "Method": "incremental",372 "Model": "Phind-CodeLlama-34B-v2",373 "Pass_at_1": 0.4878321662374 },375 {376 "Context": "selective",377 "Method": "independent",378 "Model": "Phind-CodeLlama-34B-v2",379 "Pass_at_1": 0.4863639752380 },381 {382 "Context": "selective",383 "Method": "independent",384 "Model": "WizardCoder-15B-V1.0",385 "Pass_at_1": 0.4261740357386 },387 {388 "Context": "selective",389 "Method": "incremental",390 "Model": "WizardCoder-15B-V1.0",391 "Pass_at_1": 0.3468474087392 }393 ],394 "pass_test_wise": [395 {396 "Context": "selective",397 "Method": "holistic",398 "Model": "gpt-4o-2024-05-13",399 "Pass_at_1": 0.3438179726400 },401 {402 "Context": "selective",403 "Method": "holistic",404 "Model": "deepseek-coder-33b-instruct",405 "Pass_at_1": 0.3047552867406 },407 {408 "Context": "selective",409 "Method": "holistic",410 "Model": "gpt-3.5-turbo-1106",411 "Pass_at_1": 0.2941156144412 },413 {414 "Context": "selective",415 "Method": "holistic",416 "Model": "Phind-CodeLlama-34B-v2",417 "Pass_at_1": 0.2544265255418 },419 {420 "Context": "selective",421 "Method": "independent",422 "Model": "deepseek-coder-33b-instruct",423 "Pass_at_1": 0.2224382166424 },425 {426 "Context": "selective",427 "Method": "independent",428 "Model": "deepseek-coder-6.7b-instruct",429 "Pass_at_1": 0.2083516025430 },431 {432 "Context": "selective",433 "Method": "holistic",434 "Model": "deepseek-coder-6.7b-instruct",435 "Pass_at_1": 0.2028454735436 },437 {438 "Context": "selective",439 "Method": "incremental",440 "Model": "deepseek-coder-6.7b-instruct",441 "Pass_at_1": 0.1967821219442 },443 {444 "Context": "selective",445 "Method": "independent",446 "Model": "gpt-3.5-turbo-1106",447 "Pass_at_1": 0.1930147059448 },449 {450 "Context": "selective",451 "Method": "holistic",452 "Model": "WizardCoder-15B-V1.0",453 "Pass_at_1": 0.1669267449454 },455 {456 "Context": "selective",457 "Method": "independent",458 "Model": "Phind-CodeLlama-34B-v2",459 "Pass_at_1": 0.0714792814460 },461 {462 "Context": "selective",463 "Method": "incremental",464 "Model": "Phind-CodeLlama-34B-v2",465 "Pass_at_1": 0.061012122466 },467 {468 "Context": "selective",469 "Method": "incremental",470 "Model": "gpt-3.5-turbo-1106",471 "Pass_at_1": 0.0514928193472 },473 {474 "Context": "selective",475 "Method": "incremental",476 "Model": "deepseek-coder-33b-instruct",477 "Pass_at_1": 0.0350620781478 },479 {480 "Context": "selective",481 "Method": "independent",482 "Model": "WizardCoder-15B-V1.0",483 "Pass_at_1": 0.0484 },485 {486 "Context": "selective",487 "Method": "incremental",488 "Model": "WizardCoder-15B-V1.0",489 "Pass_at_1": 0.0490 }491 ]492}