nutrientdocs/document-classification-leaderboard

1<!doctype html>2<html lang="en">3<head>4<meta charset="utf-8">5<meta name="viewport" content="width=device-width, initial-scale=1">6<title>Document Classification Leaderboard</title>7<style>8  :root{9    color-scheme:light dark;10    --paper:#eef1f4; --surface:#ffffff; --ink:#17222c; --muted:#5c6b78;11    --hair:#d7dee4; --hair-strong:#c2ccd4;12    --accent:#1d5fa8; --accent-soft:#e5eefa;13    --good:#2f8f6f; --good-soft:#dcefe8; --bad:#c0473b; --warn:#c98a2b;14    --serif:"Iowan Old Style","Palatino Linotype",Palatino,"Book Antiqua",Georgia,serif;15    --sans:system-ui,-apple-system,"Segoe UI",Roboto,Helvetica,Arial,sans-serif;16    --mono:ui-monospace,"SF Mono",SFMono-Regular,Menlo,Consolas,monospace;17  }18  @media (prefers-color-scheme:dark){19    :root{--paper:#12171c;--surface:#1a2229;--ink:#e6ecf1;--muted:#93a2ae;20      --hair:#2b353e;--hair-strong:#38444e;--accent:#6aa9e6;--accent-soft:#1e2b3a;21      --good:#57b895;--good-soft:#173029;--bad:#e0796d;--warn:#d9a856;}22  }23  *{box-sizing:border-box}24  body{margin:0;background:var(--paper);color:var(--ink);font-family:var(--sans);line-height:1.6;font-size:16px;-webkit-font-smoothing:antialiased}25  .wrap{max-width:1180px;margin:0 auto;padding:clamp(20px,5vw,60px) clamp(18px,6vw,64px)}26  .eyebrow{font-family:var(--mono);font-size:12px;letter-spacing:.18em;text-transform:uppercase;color:var(--accent);font-weight:600;margin:0 0 14px}27  h1{font-family:var(--serif);font-weight:600;font-size:clamp(28px,5vw,44px);line-height:1.08;letter-spacing:-.01em;margin:0 0 14px}28  .lede{font-size:18px;color:var(--muted);max-width:82ch;margin:0 0 8px}29  a{color:var(--accent)}30  code{font-family:var(--mono);font-size:.86em;background:var(--accent-soft);padding:.1em .38em;border-radius:4px}31  .tracknote{background:var(--surface);border:1px solid var(--hair);border-radius:12px;padding:16px 20px;margin:26px 0 6px;font-size:14px}32  .tracknote b{font-family:var(--mono);font-size:12px;letter-spacing:.04em;color:var(--accent)}33  .tracknote ul{margin:8px 0 0;padding-left:18px}.tracknote li{margin:4px 0}34  .tablebox{overflow-x:auto;margin:14px 0}35  table{border-collapse:collapse;width:100%;font-size:14px;min-width:860px}36  th,td{text-align:left;padding:10px 12px;border-bottom:1px solid var(--hair)}37  th{font-family:var(--mono);font-size:11px;letter-spacing:.06em;text-transform:uppercase;color:var(--muted);font-weight:600}38  td.n,th.n{text-align:right;font-variant-numeric:tabular-nums;font-family:var(--mono)}39  tbody tr:last-child td{border-bottom:none}40  tr.ours{background:var(--accent-soft)}41  .rank{font-family:var(--mono);color:var(--muted);width:2ch}42  .name{font-weight:650}43  .chip{display:inline-block;font-family:var(--mono);font-size:10px;font-weight:600;letter-spacing:.03em;padding:2px 7px;border-radius:20px;white-space:nowrap;margin-left:6px}44  .c-com{background:var(--accent-soft);color:var(--accent)}45  .c-open{background:var(--good-soft);color:var(--good)}46  .c-cloud{background:#f0e9dc;color:var(--warn)}47  @media (prefers-color-scheme:dark){.c-cloud{background:#2a2312;color:var(--warn)}}48  .c-gen{background:var(--hair);color:var(--muted)}49  .tabs{display:flex;gap:8px;margin:20px 0 4px}50  .tab{font-family:var(--mono);font-size:12px;font-weight:600;letter-spacing:.03em;padding:7px 14px;border-radius:8px;51    border:1px solid var(--hair-strong);background:var(--surface);color:var(--muted);cursor:pointer}52  .tab.on{background:var(--accent);color:#fff;border-color:var(--accent)}53  .oursonly{color:var(--muted)}54  .bar{position:relative;height:15px;background:var(--hair);border-radius:4px;min-width:110px}55  .bar>span{position:absolute;left:0;top:0;bottom:0;background:var(--accent);border-radius:4px}56  .bar.hl>span{background:var(--good)}57  .barnum{font-family:var(--mono);font-variant-numeric:tabular-nums;font-size:12.5px;margin-left:8px;color:var(--muted)}58  .barcell{display:flex;align-items:center;min-width:200px}59  footer{margin-top:52px;border-top:1px solid var(--hair-strong);padding-top:16px;color:var(--muted);font-size:12.5px;font-family:var(--mono)}60  footer img{vertical-align:middle;margin-right:10px;border-radius:6px}61  .muted{color:var(--muted)}62</style>63</head>64<body>65<div class="wrap">66  <p class="eyebrow">Open-vocab document classification · benchmark</p>67  <h1>Document Classification Leaderboard</h1>68  <p class="lede">Zero-shot document-type classification — labels supplied at inference, not baked into a69    head. Ranked by <strong>doc-type mean macro-F1</strong> across DocLayNet, Forms and Tobacco on the70    held-out71    <a href="https://huggingface.co/datasets/nutrientdocs/document-classification-benchmark">document-classification-benchmark</a>.72    Nutrient's in-house small models vs. cloud frontier VLMs, all scored by the same73    <code>score_classify.py</code> (unweighted macro-F1, argmax over candidate labels).</p>74 75  <div class="tracknote">76    <b>TRACKS · macro-F1</b>77    <ul>78      <li><strong>DocLayNet / Forms / Tobacco</strong> — document-type tracks (n=377 / 360 / 500); the79        headline <strong>doc-type mean</strong> averages these three and rows are ranked by it.</li>80      <li><strong>OOD</strong> (unseen types) &amp; <strong>OOV</strong> (synonym label wording) — robustness81        tracks, reported for the <em>open-vocab</em> models (Nutrient + the non-VLM generalists). Cloud VLMs82        train on ~all data, so held-out/open-vocab generalization isn't a meaningful axis for them (shown —).</li>83    </ul>84    <p style="margin:10px 0 0"><strong>Reading it:</strong> the in-house models match/lead the cloud on the85      <em>visual</em> tracks (DocLayNet, Forms) at zero API cost, and trail on <em>Tobacco</em> — a86      read-the-header task where a VLM's text reading wins. A dash (—) = track not run for that model.87      <strong>jev (text · OCR)</strong> is a text-only reference (TypeSafe): it classifies from each88      document's OCR/extracted text rather than the image — hence no OOD/OOV coverage.</p>89  </div>90 91  <div class="tabs">92    <button id="t-nonvlm" class="tab on" onclick="setTab('nonvlm')">vs open-vocab models (non-VLM)</button>93    <button id="t-vlm" class="tab" onclick="setTab('vlm')">vs cloud VLMs</button>94  </div>95  <div id="banner"></div>96  <div class="tablebox"><table><thead id="head"></thead><tbody id="rows"></tbody></table></div>97  <p class="muted" style="font-size:13px">98    <span class="chip c-com">Nutrient · commercial</span>99    <span class="chip c-open">Nutrient · open-weight</span>100    <span class="chip c-cloud">cloud VLM</span>101    &nbsp; Highlighted rows are Nutrient models. Numbers are provisional pending a clean scoring pass.</p>102 103  <!-- Plain static links so HuggingFace links these model pages (the table links are JS-rendered). -->104  <p id="models-benchmarked" class="muted" style="font-size:12.5px;margin-top:22px">105    <strong>Models benchmarked:</strong>106    <a href="https://huggingface.co/nutrientdocs/document-classification-v2">nutrientdocs/document-classification-v2</a> ·107    <a href="https://huggingface.co/nutrientdocs/document-classification-v1">nutrientdocs/document-classification-v1</a> ·108    <a href="https://huggingface.co/nutrientdocs/nutrient-document-decision">nutrientdocs/nutrient-document-decision</a> ·109    <a href="https://huggingface.co/google/siglip2-so400m-patch16-512">google/siglip2-so400m-patch16-512</a> ·110    <a href="https://huggingface.co/google/siglip2-base-patch16-512">google/siglip2-base-patch16-512</a> ·111    <a href="https://huggingface.co/google/siglip-base-patch16-224">google/siglip-base-patch16-224</a> ·112    <a href="https://huggingface.co/openai/clip-vit-large-patch14-336">openai/clip-vit-large-patch14-336</a> ·113    <a href="https://huggingface.co/openai/clip-vit-base-patch32">openai/clip-vit-base-patch32</a> ·114    <a href="https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K">laion/CLIP-ViT-H-14</a> ·115    <a href="https://huggingface.co/apple/DFN5B-CLIP-ViT-H-14-378">apple/DFN5B-CLIP-ViT-H-14-378</a> ·116    <a href="https://huggingface.co/timm/eva02_large_patch14_clip_336.merged2b_s6b_b61k">timm/eva02_large_patch14_clip_336</a>.117    Cloud VLMs (no public HF page): GPT-4o / 4o-mini / 4.1 / 4.1-mini · Gemini 2.5 Pro / Flash / Flash-Lite ·118    Claude Opus / Sonnet / Haiku ·119    TypeSafe Jev (text-only; classifies from OCR'd document text, not the image).</p>120 121  <footer>122    <a href="https://nutrient.io/"><img src="https://avatars2.githubusercontent.com/u/1527679?v=3&s=200" height="34" /></a>123    Macro-F1 on nutrientdocs/document-classification-benchmark · same <code>score_classify.py</code> for all ·124    maintained and funded by <a href="https://nutrient.io/">Nutrient</a> ·125    flagship: <a href="https://huggingface.co/nutrientdocs/document-classification-v2">document-classification-v2</a>126  </footer>127</div>128 129<script>130let DATA=[];131let TAB="nonvlm";   // default: the apples-to-apples open-vocab (non-VLM) fight132function setTab(t){TAB=t;133  document.getElementById('t-nonvlm').className='tab'+(t==='nonvlm'?' on':'');134  document.getElementById('t-vlm').className='tab'+(t==='vlm'?' on':'');135  render();}136const CORE=[["doclaynet","DocLayNet"],["forms","Forms"],["tobacco","Tobacco"]];137const EXTRA=[["ood","OOD*"],["oov","OOV*"]];138const ALL=CORE.concat(EXTRA);139function hf(m){return (m&&m.includes("/"))?"https://huggingface.co/"+m:null;}140function fmt(x){return (x===null||x===undefined)?'<span class="muted">—</span>':x.toFixed(3).replace(/^0/,"");}141function coreMean(t){const v=CORE.map(c=>t[c[0]]).filter(x=>x!==null&&x!==undefined);142  return v.length===CORE.length? v.reduce((a,b)=>a+b,0)/v.length : null;}143function chip(k){return k==="ours-commercial"?'<span class="chip c-com">commercial</span>'144  :k==="ours-open"?'<span class="chip c-open">open</span>'145  :k==="generalist"?'<span class="chip c-gen">generalist</span>':'<span class="chip c-cloud">cloud</span>';}146function render(){147  // full-coverage (all 3 core tracks) ranked by doc-type mean; partial rows after, by DocLayNet148  const D=DATA.filter(r=> TAB==="vlm" ? r.kind!=="generalist" : r.kind!=="cloud");  // ours anchor both tabs149  const full=D.filter(r=>coreMean(r.tracks)!==null).sort((a,b)=>coreMean(b.tracks)-coreMean(a.tracks));150  const part=D.filter(r=>coreMean(r.tracks)===null).sort((a,b)=>(b.tracks.doclaynet||0)-(a.tracks.doclaynet||0));151  const rows=full.concat(part);152  document.getElementById("head").innerHTML='<tr><th class="rank">#</th><th>Model</th>'153    +ALL.map(t=>`<th class="n">${t[1]}</th>`).join("")+'<th class="barcell">Doc-type mean</th><th class="n">Latency</th></tr>';154  const tb=document.getElementById("rows");tb.innerHTML="";155  const max=Math.max(...full.map(r=>coreMean(r.tracks)));156  rows.forEach((r,i)=>{157    const m=coreMean(r.tracks), ours=r.kind&&r.kind.startsWith("ours");158    const tr=document.createElement("tr");if(ours)tr.className="ours";159    const nm=hf(r.model)?`<a class="name" href="${hf(r.model)}" target="_blank" rel="noopener">${r.name}</a>`:`<span class="name">${r.name}</span>`;160    tr.innerHTML=`<td class="rank">${m!==null?i+1:''}</td><td>${nm}${chip(r.chip_kind||r.kind)}</td>`161      +ALL.map(t=>`<td class="n ${EXTRA.some(e=>e[0]===t[0])?'oursonly':''}">${fmt(r.tracks[t[0]])}</td>`).join("")162      +(m!==null?`<td><div class="barcell"><div class="bar ${ours?'hl':''}" style="flex:1"><span style="width:${Math.round(m/max*100)}%"></span></div><span class="barnum">${m.toFixed(3).replace(/^0/,"")}</span></div></td>`163        :`<td class="muted" style="font-size:12px">partial coverage</td>`)164      +`<td class="n">${r.latency||'<span class="muted">—</span>'}</td>`;165    tb.appendChild(tr);166  });167}168function setBanner(){const b=document.getElementById("banner");169  if(DATA.some(d=>d.provisional)){b.className="tracknote";b.style.borderColor="var(--warn)";b.textContent="Provisional numbers — a clean scoring pass on standard hardware is pending.";}}170fetch("leaderboard.json").then(r=>r.ok?r.json():[]).then(d=>{DATA=Array.isArray(d)?d:(d.results||[]);render();}).catch(()=>render());171</script>172</body>173</html>174