TextCortex/clef-cybersecurity

1{2  "type": "aggregate_prompt_injection_benchmarks",3  "models": [4    {5      "model": "TextCortex/clef-cybersecurity",6      "name": "clef-cybersecurity",7      "full_en": {8        "n": 510,9        "attacks": 256,10        "clean": 254,11        "auroc": 0.9925335260826772,12        "caught": 232,13        "false_alarms": 014      },15      "full_de": {16        "n": 510,17        "attacks": 256,18        "clean": 254,19        "auroc": 0.9743940698818898,20        "caught": 241,21        "false_alarms": 1722      },23      "skills_en": {24        "n": 48,25        "attacks": 27,26        "clean": 21,27        "auroc": 1.0,28        "caught": 23,29        "false_alarms": 030      },31      "skills_de": {32        "n": 48,33        "attacks": 27,34        "clean": 21,35        "auroc": 0.9171075837742504,36        "caught": 26,37        "false_alarms": 338      },39      "pdf": {40        "n": 730,41        "attacks": 107,42        "clean": 623,43        "auroc": 0.9856212778086137,44        "caught": 84,45        "false_alarms": 346      },47      "decision_rule": "score > 0.5"48    },49    {50      "model": "Cloudflare/clef-flash",51      "name": "CLEF Flash (base)",52      "full_en": {53        "n": 510,54        "attacks": 256,55        "clean": 254,56        "auroc": 0.9587613804133859,57        "caught": 136,58        "false_alarms": 059      },60      "full_de": {61        "n": 510,62        "attacks": 256,63        "clean": 254,64        "auroc": 0.9391455462598425,65        "caught": 130,66        "false_alarms": 067      },68      "skills_en": {69        "n": 48,70        "attacks": 27,71        "clean": 21,72        "auroc": 0.9761904761904762,73        "caught": 14,74        "false_alarms": 075      },76      "skills_de": {77        "n": 48,78        "attacks": 27,79        "clean": 21,80        "auroc": 0.9047619047619048,81        "caught": 12,82        "false_alarms": 083      },84      "pdf": {85        "n": 730,86        "attacks": 107,87        "clean": 623,88        "auroc": 0.8144492281843957,89        "caught": 6,90        "false_alarms": 091      },92      "decision_rule": "score > 0.5"93    },94    {95      "model": "jev",96      "name": "Jev",97      "full_en": {98        "auroc": 0.9799920029527559,99        "n": 510100      },101      "full_de": {102        "auroc": 0.9564391609251969,103        "n": 510104      },105      "skills_en": {106        "n": 48,107        "attacks": 27,108        "clean": 21,109        "auroc": 0.9841269841269841,110        "caught": 25,111        "false_alarms": 1112      },113      "skills_de": {114        "n": 48,115        "attacks": 27,116        "clean": 21,117        "auroc": 0.9603174603174603,118        "caught": 25,119        "false_alarms": 2120      },121      "pdf": {122        "n": 730,123        "attacks": 107,124        "clean": 623,125        "auroc": 0.978525674682348,126        "caught": 73,127        "false_alarms": 2128      },129      "decision_rule": "score > 0.5"130    },131    {132      "model": "laya-cybersec-r2a",133      "name": "Laya R2a",134      "full_en": {135        "n": 510,136        "attacks": 256,137        "clean": 254,138        "auroc": 0.9155004306102362,139        "caught": 140,140        "false_alarms": 2,141        "recall": 0.546875,142        "fpr": 0.007874015748031496143      },144      "full_de": {145        "n": 510,146        "attacks": 256,147        "clean": 254,148        "auroc": 0.8780065821850394,149        "caught": 151,150        "false_alarms": 9,151        "recall": 0.58984375,152        "fpr": 0.03543307086614173153      },154      "skills_en": {155        "n": 48,156        "attacks": 27,157        "clean": 21,158        "auroc": 0.927689594356261,159        "caught": 18,160        "false_alarms": 0161      },162      "skills_de": {163        "n": 48,164        "attacks": 27,165        "clean": 21,166        "auroc": 0.9329805996472663,167        "caught": 18,168        "false_alarms": 0169      },170      "pdf": {171        "n": 730,172        "attacks": 107,173        "clean": 623,174        "auroc": 0.8855927753859077,175        "caught": 81,176        "false_alarms": 0177      },178      "decision_rule": "score > 0.95"179    }180  ],181  "cohort": {182    "full_per_language": 510,183    "skills_per_language": 48,184    "pdf_attacks": 107,185    "clean_pdfs": 623186  },187  "scope": "Previously inspected internal regression sets; not a public leaderboard or fresh blind test.",188  "threshold_note": "Strict >0.5 for both CLEF models and Jev; strict >0.95 for Laya R2a. Recall and false alarms are not equal-threshold comparisons.",189  "base_clef_evaluation": {190    "type": "unchanged_local_base",191    "model": "Cloudflare/clef-flash",192    "revision": "17f0b0ad64efb65d273590632833508766b2aae6",193    "scope": "Unchanged native checkpoint, evaluated locally on the matched cohorts with token-bounded windows. Not the Cloudflare hosted API run."194  },195  "selected_epoch": 3,196  "epochs_trained": 4,197  "clef_secondary_pdf_at_095": {198    "n": 730,199    "attacks": 107,200    "clean": 623,201    "auroc": 0.9856212778086137,202    "caught": 69,203    "false_alarms": 2204  }205}206