typesafe/evalsafe-invoice-processing
Invoice processing Snapshot: 2026-09-28. 150 cases and 6,874 question instances. Default reference: consensus. Labels are model-generated references. Data Load configuration cases, questions, or run_results; all have a test split. cases: one row per case_id, with the complete input in input_json, descriptive metadata_json, and openai, anthropic, and consensus labelsets. Decisions are grouped by policy_id and contain status, actions, and primary_action. questions:… See the full description on the dataset page: https://huggingface.co/datasets/typesafe/evalsafe-invoice-processing.
61.3k
1{2 "license": "apache-2.0",3 "title": "Invoice processing",4 "workflow": "invoice_processing",5 "snapshot_date": "2026-09-28",6 "default_reference": "consensus",7 "labelsets": [8 "openai",9 "anthropic",10 "consensus"11 ],12 "policies": [13 {14 "policy_id": "startup",15 "title": "startup"16 },17 {18 "policy_id": "enterprise",19 "title": "enterprise"20 },21 {22 "policy_id": "high_volume_retailer",23 "title": "high volume retailer"24 }25 ],26 "metrics": [27 {28 "metric_id": "accuracy",29 "title": "Exact action-set accuracy",30 "comparison": "exact_actions"31 },32 {33 "metric_id": "primary_match",34 "title": "Primary action agreement",35 "comparison": "primary_action"36 }37 ],38 "blend": {39 "method": "mean_probabilities",40 "missing_source": "use_available_source",41 "question_alignment": "same case, node, question definition and state"42 },43 "coverage": {44 "cases": 150,45 "questions": 6874,46 "openai_cases": 150,47 "openai_opus_fallback_cases": 0,48 "anthropic_cases": 150,49 "anthropic_opus_fallback_cases": 0,50 "consensus_cases": 150,51 "consensus_opus_fallback_cases": 0,52 "runs": 9,53 "run_results": 1350,54 "run_questions": 6186655 },56 "encodings": {57 "*_json": "JSON-encoded UTF-8 text",58 "probabilities": "list of option/probability pairs"59 },60 "runs": {61 "code-anthropic-claude-haiku-4-5-default-1bb8309e2727": {62 "model": {63 "name": "claude-haiku-4-5",64 "provider": "anthropic",65 "reasoning_effort": "default",66 "question_mode": "all"67 },68 "cases": 150,69 "decisions": 450,70 "questions": 6874,71 "scores": [72 {73 "metric_id": "accuracy",74 "value": 0.4288888888888889,75 "denominator": 45076 },77 {78 "metric_id": "primary_match",79 "value": 0.7177777777777777,80 "denominator": 45081 }82 ],83 "mean_cost_usd": 0.055838246666666674,84 "cost_cases": 150,85 "mean_wall_time_s": 30.751862279434377,86 "wall_time_cases": 150,87 "mean_summed_call_time_s": 30.309142640261754,88 "summed_call_time_cases": 150,89 "cost_bases": [90 "estimated_uncached"91 ]92 },93 "code-anthropic-claude-opus-5-default-47df2a4b737c": {94 "model": {95 "name": "claude-opus-5",96 "provider": "anthropic",97 "reasoning_effort": "default",98 "question_mode": "all"99 },100 "cases": 150,101 "decisions": 450,102 "questions": 6874,103 "scores": [104 {105 "metric_id": "accuracy",106 "value": 0.7844444444444445,107 "denominator": 450108 },109 {110 "metric_id": "primary_match",111 "value": 0.8888888888888888,112 "denominator": 450113 }114 ],115 "mean_cost_usd": 0.4856332333333333,116 "cost_cases": 150,117 "mean_wall_time_s": 92.05054310917699,118 "wall_time_cases": 150,119 "mean_summed_call_time_s": 91.54910127477255,120 "summed_call_time_cases": 150,121 "cost_bases": [122 "estimated_uncached"123 ]124 },125 "code-anthropic-claude-sonnet-5-default-1312a6116364": {126 "model": {127 "name": "claude-sonnet-5",128 "provider": "anthropic",129 "reasoning_effort": "default",130 "question_mode": "all"131 },132 "cases": 150,133 "decisions": 450,134 "questions": 6874,135 "scores": [136 {137 "metric_id": "accuracy",138 "value": 0.7288888888888889,139 "denominator": 450140 },141 {142 "metric_id": "primary_match",143 "value": 0.8644444444444445,144 "denominator": 450145 }146 ],147 "mean_cost_usd": 0.36156889333333336,148 "cost_cases": 150,149 "mean_wall_time_s": 241.2727579686612,150 "wall_time_cases": 150,151 "mean_summed_call_time_s": 240.59865719914515,152 "summed_call_time_cases": 150,153 "cost_bases": [154 "estimated_uncached"155 ]156 },157 "code-fireworks-deepseek-v4-flash-0731-default-e48390bdfd24": {158 "model": {159 "name": "accounts/fireworks/models/deepseek-v4-flash-0731",160 "provider": "fireworks",161 "reasoning_effort": "default",162 "question_mode": "all"163 },164 "cases": 150,165 "decisions": 450,166 "questions": 6874,167 "scores": [168 {169 "metric_id": "accuracy",170 "value": 0.6977777777777778,171 "denominator": 450172 },173 {174 "metric_id": "primary_match",175 "value": 0.8844444444444445,176 "denominator": 450177 }178 ],179 "mean_cost_usd": 0.013302112266666668,180 "cost_cases": 150,181 "mean_wall_time_s": 84.06482463536085,182 "wall_time_cases": 150,183 "mean_summed_call_time_s": 84.06207437168729,184 "summed_call_time_cases": 150,185 "cost_bases": [186 "estimated_uncached"187 ]188 },189 "code-fireworks-deepseek-v4-pro-0813-default-e80ab7fa20e7": {190 "model": {191 "name": "accounts/fireworks/models/deepseek-v4-pro-0813",192 "provider": "fireworks",193 "reasoning_effort": "default",194 "question_mode": "all"195 },196 "cases": 150,197 "decisions": 450,198 "questions": 6874,199 "scores": [200 {201 "metric_id": "accuracy",202 "value": 0.7266666666666667,203 "denominator": 450204 },205 {206 "metric_id": "primary_match",207 "value": 0.88,208 "denominator": 450209 }210 ],211 "mean_cost_usd": 0.0830030608,212 "cost_cases": 150,213 "mean_wall_time_s": 137.40619414747306,214 "wall_time_cases": 150,215 "mean_summed_call_time_s": 137.40277723060146,216 "summed_call_time_cases": 150,217 "cost_bases": [218 "estimated_uncached"219 ]220 },221 "code-openai-gpt-5.6-luna-default-ba70c917ae44": {222 "model": {223 "name": "gpt-5.6-luna",224 "provider": "openai",225 "reasoning_effort": "default",226 "question_mode": "all"227 },228 "cases": 150,229 "decisions": 450,230 "questions": 6874,231 "scores": [232 {233 "metric_id": "accuracy",234 "value": 0.6777777777777778,235 "denominator": 450236 },237 {238 "metric_id": "primary_match",239 "value": 0.8511111111111112,240 "denominator": 450241 }242 ],243 "mean_cost_usd": 0.008134037333333333,244 "cost_cases": 150,245 "mean_wall_time_s": 21.41112150724434,246 "wall_time_cases": 150,247 "mean_summed_call_time_s": 21.40855329136209,248 "summed_call_time_cases": 150,249 "cost_bases": [250 "estimated_uncached"251 ]252 },253 "code-openai-gpt-5.6-sol-default-b1da13240cc3": {254 "model": {255 "name": "gpt-5.6-sol",256 "provider": "openai",257 "reasoning_effort": "default",258 "question_mode": "all"259 },260 "cases": 150,261 "decisions": 450,262 "questions": 6874,263 "scores": [264 {265 "metric_id": "accuracy",266 "value": 0.7911111111111111,267 "denominator": 450268 },269 {270 "metric_id": "primary_match",271 "value": 0.9,272 "denominator": 450273 }274 ],275 "mean_cost_usd": 0.2151939333333333,276 "cost_cases": 150,277 "mean_wall_time_s": 34.27234422469589,278 "wall_time_cases": 150,279 "mean_summed_call_time_s": 34.26975542746251,280 "summed_call_time_cases": 150,281 "cost_bases": [282 "estimated_uncached"283 ]284 },285 "code-openai-gpt-5.6-terra-default-bb284449d63b": {286 "model": {287 "name": "gpt-5.6-terra",288 "provider": "openai",289 "reasoning_effort": "default",290 "question_mode": "all"291 },292 "cases": 150,293 "decisions": 450,294 "questions": 6874,295 "scores": [296 {297 "metric_id": "accuracy",298 "value": 0.7466666666666667,299 "denominator": 450300 },301 {302 "metric_id": "primary_match",303 "value": 0.8933333333333333,304 "denominator": 450305 }306 ],307 "mean_cost_usd": 0.07780061333333334,308 "cost_cases": 150,309 "mean_wall_time_s": 17.319832631626632,310 "wall_time_cases": 150,311 "mean_summed_call_time_s": 17.317260218905286,312 "summed_call_time_cases": 150,313 "cost_bases": [314 "estimated_uncached"315 ]316 },317 "code-typesafe-jev-1.13.0-off-160259a2ba80": {318 "model": {319 "name": "jev-1.13.0",320 "provider": "typesafe",321 "reasoning_effort": "off",322 "question_mode": "all"323 },324 "cases": 150,325 "decisions": 450,326 "questions": 6874,327 "scores": [328 {329 "metric_id": "accuracy",330 "value": 0.6177777777777778,331 "denominator": 450332 },333 {334 "metric_id": "primary_match",335 "value": 0.8311111111111111,336 "denominator": 450337 }338 ],339 "mean_cost_usd": 0.0010994398800000002,340 "cost_cases": 150,341 "mean_wall_time_s": 0.4978434846914994,342 "wall_time_cases": 150,343 "mean_summed_call_time_s": 0.4951227500002521,344 "summed_call_time_cases": 150,345 "cost_bases": [346 "estimated_uncached"347 ]348 }349 },350 "run_results": {351 "route": "code",352 "reference": "consensus",353 "primary_metric": "accuracy",354 "selection": "Final plots: highest primary accuracy per selected model configuration.",355 "overall_aggregation": "Equal workflow means for configurations represented in all four workflows."356 }357}358