CPunisher/JavaBench
1
1{2 "completion": [3 {4 "Context": "selective",5 "Method": "holistic",6 "Model": "gpt-4o-2024-05-13",7 "Pass_at_1": 1.08 },9 {10 "Context": "maximum",11 "Method": "holistic",12 "Model": "deepseek-coder-33b-instruct",13 "Pass_at_1": 0.971428571414 },15 {16 "Context": "maximum",17 "Method": "holistic",18 "Model": "deepseek-coder-6.7b-instruct",19 "Pass_at_1": 0.94520 },21 {22 "Context": "selective",23 "Method": "holistic",24 "Model": "deepseek-coder-6.7b-instruct",25 "Pass_at_1": 0.937857142926 },27 {28 "Context": "selective",29 "Method": "holistic",30 "Model": "deepseek-coder-33b-instruct",31 "Pass_at_1": 0.935714285732 },33 {34 "Context": "maximum",35 "Method": "holistic",36 "Model": "gpt-3.5-turbo-1106",37 "Pass_at_1": 0.932857142938 },39 {40 "Context": "selective",41 "Method": "holistic",42 "Model": "gpt-3.5-turbo-1106",43 "Pass_at_1": 0.921428571444 },45 {46 "Context": "minimum",47 "Method": "holistic",48 "Model": "deepseek-coder-6.7b-instruct",49 "Pass_at_1": 0.900714285750 },51 {52 "Context": "maximum",53 "Method": "holistic",54 "Model": "Phind-CodeLlama-34B-v2",55 "Pass_at_1": 0.890714285756 },57 {58 "Context": "minimum",59 "Method": "holistic",60 "Model": "deepseek-coder-33b-instruct",61 "Pass_at_1": 0.882857142962 },63 {64 "Context": "minimum",65 "Method": "holistic",66 "Model": "Phind-CodeLlama-34B-v2",67 "Pass_at_1": 0.872857142968 },69 {70 "Context": "minimum",71 "Method": "holistic",72 "Model": "gpt-3.5-turbo-1106",73 "Pass_at_1": 0.857857142974 },75 {76 "Context": "selective",77 "Method": "holistic",78 "Model": "Phind-CodeLlama-34B-v2",79 "Pass_at_1": 0.84580 },81 {82 "Context": "selective",83 "Method": "holistic",84 "Model": "WizardCoder-15B-V1.0",85 "Pass_at_1": 0.796428571486 },87 {88 "Context": "minimum",89 "Method": "holistic",90 "Model": "WizardCoder-15B-V1.0",91 "Pass_at_1": 0.793571428692 },93 {94 "Context": "maximum",95 "Method": "holistic",96 "Model": "WizardCoder-15B-V1.0",97 "Pass_at_1": 0.719285714398 }99 ],100 "compilation_class_wise": [101 {102 "Context": "selective",103 "Method": "holistic",104 "Model": "gpt-3.5-turbo-1106",105 "Pass_at_1": 0.7942857143106 },107 {108 "Context": "selective",109 "Method": "holistic",110 "Model": "deepseek-coder-33b-instruct",111 "Pass_at_1": 0.7414285714112 },113 {114 "Context": "maximum",115 "Method": "holistic",116 "Model": "deepseek-coder-6.7b-instruct",117 "Pass_at_1": 0.7385714286118 },119 {120 "Context": "maximum",121 "Method": "holistic",122 "Model": "deepseek-coder-33b-instruct",123 "Pass_at_1": 0.7314285714124 },125 {126 "Context": "selective",127 "Method": "holistic",128 "Model": "deepseek-coder-6.7b-instruct",129 "Pass_at_1": 0.7171428571130 },131 {132 "Context": "selective",133 "Method": "holistic",134 "Model": "Phind-CodeLlama-34B-v2",135 "Pass_at_1": 0.6978571429136 },137 {138 "Context": "selective",139 "Method": "holistic",140 "Model": "gpt-4o-2024-05-13",141 "Pass_at_1": 0.6607142857142 },143 {144 "Context": "maximum",145 "Method": "holistic",146 "Model": "gpt-3.5-turbo-1106",147 "Pass_at_1": 0.6592857143148 },149 {150 "Context": "maximum",151 "Method": "holistic",152 "Model": "Phind-CodeLlama-34B-v2",153 "Pass_at_1": 0.6414285714154 },155 {156 "Context": "selective",157 "Method": "holistic",158 "Model": "WizardCoder-15B-V1.0",159 "Pass_at_1": 0.58160 },161 {162 "Context": "maximum",163 "Method": "holistic",164 "Model": "WizardCoder-15B-V1.0",165 "Pass_at_1": 0.4814285714166 },167 {168 "Context": "minimum",169 "Method": "holistic",170 "Model": "gpt-3.5-turbo-1106",171 "Pass_at_1": 0.45172 },173 {174 "Context": "minimum",175 "Method": "holistic",176 "Model": "Phind-CodeLlama-34B-v2",177 "Pass_at_1": 0.39178 },179 {180 "Context": "minimum",181 "Method": "holistic",182 "Model": "deepseek-coder-6.7b-instruct",183 "Pass_at_1": 0.3692857143184 },185 {186 "Context": "minimum",187 "Method": "holistic",188 "Model": "deepseek-coder-33b-instruct",189 "Pass_at_1": 0.3457142857190 },191 {192 "Context": "minimum",193 "Method": "holistic",194 "Model": "WizardCoder-15B-V1.0",195 "Pass_at_1": 0.215196 }197 ],198 "compilation_test_wise": [199 {200 "Context": "selective",201 "Method": "holistic",202 "Model": "gpt-4o-2024-05-13",203 "Pass_at_1": 0.5035714286204 },205 {206 "Context": "selective",207 "Method": "holistic",208 "Model": "deepseek-coder-33b-instruct",209 "Pass_at_1": 0.4202826585210 },211 {212 "Context": "selective",213 "Method": "holistic",214 "Model": "gpt-3.5-turbo-1106",215 "Pass_at_1": 0.3443277311216 },217 {218 "Context": "maximum",219 "Method": "holistic",220 "Model": "deepseek-coder-33b-instruct",221 "Pass_at_1": 0.3405987395222 },223 {224 "Context": "maximum",225 "Method": "holistic",226 "Model": "deepseek-coder-6.7b-instruct",227 "Pass_at_1": 0.3387079832228 },229 {230 "Context": "selective",231 "Method": "holistic",232 "Model": "Phind-CodeLlama-34B-v2",233 "Pass_at_1": 0.3183823529234 },235 {236 "Context": "selective",237 "Method": "holistic",238 "Model": "deepseek-coder-6.7b-instruct",239 "Pass_at_1": 0.3121848739240 },241 {242 "Context": "maximum",243 "Method": "holistic",244 "Model": "gpt-3.5-turbo-1106",245 "Pass_at_1": 0.2858193277246 },247 {248 "Context": "maximum",249 "Method": "holistic",250 "Model": "Phind-CodeLlama-34B-v2",251 "Pass_at_1": 0.2283088235252 },253 {254 "Context": "selective",255 "Method": "holistic",256 "Model": "WizardCoder-15B-V1.0",257 "Pass_at_1": 0.2240546218258 },259 {260 "Context": "maximum",261 "Method": "holistic",262 "Model": "WizardCoder-15B-V1.0",263 "Pass_at_1": 0.1466911765264 },265 {266 "Context": "minimum",267 "Method": "holistic",268 "Model": "deepseek-coder-33b-instruct",269 "Pass_at_1": 0.1128676471270 },271 {272 "Context": "minimum",273 "Method": "holistic",274 "Model": "gpt-3.5-turbo-1106",275 "Pass_at_1": 0.03125276 },277 {278 "Context": "minimum",279 "Method": "holistic",280 "Model": "WizardCoder-15B-V1.0",281 "Pass_at_1": 0.0147058824282 },283 {284 "Context": "minimum",285 "Method": "holistic",286 "Model": "deepseek-coder-6.7b-instruct",287 "Pass_at_1": 0.0125288 },289 {290 "Context": "minimum",291 "Method": "holistic",292 "Model": "Phind-CodeLlama-34B-v2",293 "Pass_at_1": 0.0294 }295 ],296 "pass_class_wise": [297 {298 "Context": "selective",299 "Method": "holistic",300 "Model": "gpt-3.5-turbo-1106",301 "Pass_at_1": 0.7832360347302 },303 {304 "Context": "selective",305 "Method": "holistic",306 "Model": "deepseek-coder-33b-instruct",307 "Pass_at_1": 0.723699056308 },309 {310 "Context": "maximum",311 "Method": "holistic",312 "Model": "deepseek-coder-33b-instruct",313 "Pass_at_1": 0.715291943314 },315 {316 "Context": "maximum",317 "Method": "holistic",318 "Model": "deepseek-coder-6.7b-instruct",319 "Pass_at_1": 0.7033228696320 },321 {322 "Context": "selective",323 "Method": "holistic",324 "Model": "deepseek-coder-6.7b-instruct",325 "Pass_at_1": 0.6855203826326 },327 {328 "Context": "selective",329 "Method": "holistic",330 "Model": "Phind-CodeLlama-34B-v2",331 "Pass_at_1": 0.6808480861332 },333 {334 "Context": "selective",335 "Method": "holistic",336 "Model": "gpt-4o-2024-05-13",337 "Pass_at_1": 0.6545897285338 },339 {340 "Context": "maximum",341 "Method": "holistic",342 "Model": "gpt-3.5-turbo-1106",343 "Pass_at_1": 0.6417690022344 },345 {346 "Context": "maximum",347 "Method": "holistic",348 "Model": "Phind-CodeLlama-34B-v2",349 "Pass_at_1": 0.6293667264350 },351 {352 "Context": "selective",353 "Method": "holistic",354 "Model": "WizardCoder-15B-V1.0",355 "Pass_at_1": 0.5674101922356 },357 {358 "Context": "maximum",359 "Method": "holistic",360 "Model": "WizardCoder-15B-V1.0",361 "Pass_at_1": 0.4741970721362 },363 {364 "Context": "minimum",365 "Method": "holistic",366 "Model": "gpt-3.5-turbo-1106",367 "Pass_at_1": 0.4489821662368 },369 {370 "Context": "minimum",371 "Method": "holistic",372 "Model": "Phind-CodeLlama-34B-v2",373 "Pass_at_1": 0.3864123669374 },375 {376 "Context": "minimum",377 "Method": "holistic",378 "Model": "deepseek-coder-6.7b-instruct",379 "Pass_at_1": 0.3599216366380 },381 {382 "Context": "minimum",383 "Method": "holistic",384 "Model": "deepseek-coder-33b-instruct",385 "Pass_at_1": 0.3272885064386 },387 {388 "Context": "minimum",389 "Method": "holistic",390 "Model": "WizardCoder-15B-V1.0",391 "Pass_at_1": 0.2139054163392 }393 ],394 "pass_test_wise": [395 {396 "Context": "selective",397 "Method": "holistic",398 "Model": "gpt-4o-2024-05-13",399 "Pass_at_1": 0.3438179726400 },401 {402 "Context": "selective",403 "Method": "holistic",404 "Model": "deepseek-coder-33b-instruct",405 "Pass_at_1": 0.3047552867406 },407 {408 "Context": "maximum",409 "Method": "holistic",410 "Model": "deepseek-coder-33b-instruct",411 "Pass_at_1": 0.3032497787412 },413 {414 "Context": "selective",415 "Method": "holistic",416 "Model": "gpt-3.5-turbo-1106",417 "Pass_at_1": 0.2941156144418 },419 {420 "Context": "selective",421 "Method": "holistic",422 "Model": "Phind-CodeLlama-34B-v2",423 "Pass_at_1": 0.2544265255424 },425 {426 "Context": "maximum",427 "Method": "holistic",428 "Model": "gpt-3.5-turbo-1106",429 "Pass_at_1": 0.2393000344430 },431 {432 "Context": "selective",433 "Method": "holistic",434 "Model": "deepseek-coder-6.7b-instruct",435 "Pass_at_1": 0.2028454735436 },437 {438 "Context": "maximum",439 "Method": "holistic",440 "Model": "Phind-CodeLlama-34B-v2",441 "Pass_at_1": 0.1966660863442 },443 {444 "Context": "maximum",445 "Method": "holistic",446 "Model": "deepseek-coder-6.7b-instruct",447 "Pass_at_1": 0.1877858469448 },449 {450 "Context": "selective",451 "Method": "holistic",452 "Model": "WizardCoder-15B-V1.0",453 "Pass_at_1": 0.1669267449454 },455 {456 "Context": "maximum",457 "Method": "holistic",458 "Model": "WizardCoder-15B-V1.0",459 "Pass_at_1": 0.1370849195460 },461 {462 "Context": "minimum",463 "Method": "holistic",464 "Model": "deepseek-coder-33b-instruct",465 "Pass_at_1": 0.1123372221466 },467 {468 "Context": "minimum",469 "Method": "holistic",470 "Model": "gpt-3.5-turbo-1106",471 "Pass_at_1": 0.03125472 },473 {474 "Context": "minimum",475 "Method": "holistic",476 "Model": "WizardCoder-15B-V1.0",477 "Pass_at_1": 0.0147058824478 },479 {480 "Context": "minimum",481 "Method": "holistic",482 "Model": "deepseek-coder-6.7b-instruct",483 "Pass_at_1": 0.0125484 },485 {486 "Context": "minimum",487 "Method": "holistic",488 "Model": "Phind-CodeLlama-34B-v2",489 "Pass_at_1": 0.0490 }491 ]492}