vireshk/LLM-Inference-Optimization

1<!doctype html>2<html lang="en">3<head>4<meta charset="utf-8">5<meta name="viewport" content="width=device-width, initial-scale=1">6<title>LLM Inference Optimization — results explorer</title>7<meta name="description" content="Open-loop benchmark results for quantized Llama-3.1-8B serving on an NVIDIA A40.">8<script src="https://cdnjs.cloudflare.com/ajax/libs/plotly.js/2.35.0/plotly.min.js"9        onerror="window.__plotlyFailed = true"></script>10<style>11  :root {12    --bg: #fcfcfb; --surface: #ffffff; --text: #0b0b0b; --muted: #52514e;13    --grid: #e3e2df; --accent: #2a78d6; --good: #1baf7a; --bad: #e34948;14  }15  @media (prefers-color-scheme: dark) {16    :root:not([data-theme="light"]) {17      --bg: #14140f; --surface: #1c1c18; --text: #f5f4ef; --muted: #a8a69e;18      --grid: #33322c; --accent: #6aa9ee; --good: #4fd0a0; --bad: #f08886;19    }20  }21  * { box-sizing: border-box; }22  body { margin: 0; background: var(--bg); color: var(--text);23         font: 15px/1.6 -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; }24  .wrap { max-width: 1080px; margin: 0 auto; padding: 40px 22px 72px; }25  h1 { font-size: 28px; line-height: 1.25; margin: 0 0 8px; letter-spacing: -0.01em; }26  h2 { font-size: 18px; margin: 40px 0 10px; letter-spacing: -0.005em; }27  .lede { color: var(--muted); margin: 0 0 6px; max-width: 68ch; }28  .links { margin: 18px 0 0; font-size: 14px; }29  .links a { color: var(--accent); text-decoration: none; margin-right: 18px; }30  .links a:hover { text-decoration: underline; }31  .panel { background: var(--surface); border: 1px solid var(--grid);32           border-radius: 10px; padding: 20px; margin-top: 18px; }33  .sla-row { display: flex; align-items: center; gap: 16px; flex-wrap: wrap; }34  .sla-row label { font-weight: 600; white-space: nowrap; }35  input[type=range] { flex: 1 1 260px; min-width: 200px; accent-color: var(--accent); }36  .budget { font-variant-numeric: tabular-nums; font-weight: 600; min-width: 5.5em; }37  .verdict { margin-top: 16px; font-size: 15px; line-height: 1.7; }38  .verdict strong { font-size: 17px; }39  code { font-family: ui-monospace, SFMono-Regular, Menlo, monospace; font-size: 0.92em;40         background: var(--bg); border: 1px solid var(--grid); border-radius: 4px; padding: 1px 5px; }41  .scroll { overflow-x: auto; -webkit-overflow-scrolling: touch; }42  table { border-collapse: collapse; width: 100%; font-size: 14px; min-width: 640px; }43  th, td { text-align: right; padding: 7px 10px; border-bottom: 1px solid var(--grid);44           font-variant-numeric: tabular-nums; white-space: nowrap; }45  th:first-child, td:first-child { text-align: left; font-variant-numeric: normal; }46  th { color: var(--muted); font-weight: 600; font-size: 12.5px; text-transform: uppercase;47       letter-spacing: 0.04em; }48  tr.out td { opacity: 0.32; }49  tr.best td { background: color-mix(in srgb, var(--good) 12%, transparent); font-weight: 600; }50  .tag { display: inline-block; font-size: 11.5px; padding: 1px 7px; border-radius: 999px;51         border: 1px solid currentColor; }52  .tag.in { color: var(--good); } .tag.out { color: var(--muted); }53  .note { color: var(--muted); font-size: 13.5px; margin-top: 12px; max-width: 74ch; }54  #chart { width: 100%; height: 460px; }55  footer { margin-top: 48px; padding-top: 20px; border-top: 1px solid var(--grid);56           color: var(--muted); font-size: 13.5px; }57</style>58</head>59<body>60<div class="wrap">61 62  <h1>Quantized LLM serving on an NVIDIA A40</h1>63  <p class="lede" id="lede">64    Llama-3.1-8B-Instruct &middot; open-loop Poisson arrivals, repeated measurements,65    clocks locked, kernels asserted from engine logs.66  </p>67  <p class="lede">68    Every point below is a rate a server could actually be provisioned for. Oversubscribed69    runs &mdash; where offered load exceeded capacity and latency reflects run duration rather70    than load &mdash; are excluded here and published in the repository.71  </p>72  <p class="links">73    <a href="https://github.com/vireshkoli/LLM-Inference-Optimization">Repository</a>74    <a href="https://github.com/vireshkoli/LLM-Inference-Optimization/blob/main/REPORT.md">Findings</a>75    <a href="https://github.com/vireshkoli/LLM-Inference-Optimization/blob/main/METHODOLOGY.md">Methodology</a>76  </p>77 78  <h2>Pick a latency budget</h2>79  <div class="panel">80    <div class="sla-row">81      <label for="sla">p95 TTFT budget</label>82      <input type="range" id="sla" min="100" max="3000" step="25" value="500">83      <span class="budget" id="slaVal">500 ms</span>84    </div>85    <div class="verdict" id="verdict"></div>86    <p class="note">87      Because every configuration ran on the same GPU, the hourly price is a linear scalar and88      cannot reorder this ranking. The latency budget can, and does &mdash; it decides which89      offered rates are admissible, which caps sustainable throughput, which sets cost per token.90    </p>91  </div>92 93  <h2>Latency vs throughput</h2>94  <div class="panel">95    <div id="chart"></div>96    <p class="note">97      Click a legend entry to hide a configuration. The dashed line is the current budget;98      points above it do not meet the SLA. Log scale on the y axis, because tail latency spans99      orders of magnitude between the linear region and saturation.100    </p>101  </div>102 103  <h2>Operating points</h2>104  <div class="panel">105    <div class="scroll"><table id="tbl">106      <thead><tr>107        <th>Config</th><th>Rate</th><th>Throughput</th><th>TTFT p95</th>108        <th>TPOT p95</th><th>$/1M tokens</th><th>Repeats</th><th>SLA</th>109      </tr></thead>110      <tbody></tbody>111    </table></div>112    <p class="note">113      Throughput is mean across repeats. Cost assumes <span id="price">the reference price</span>114      per GPU-hour from <code>configs/cost.yaml</code> &mdash; an assumption, stated as one,115      because the measurement GPU is a lab machine with no invoice.116    </p>117  </div>118 119  <footer>120    Generated from <code>results/runs/*.json</code> by <code>make report</code>. Every number on121    this page is computed from that data as the page loads. Full methodology, known confounds and122    limitations are in the repository.123  </footer>124</div>125 126<script>127// Fixed palette assigned by configuration identity, never by rank, so hiding a128// series never repaints the survivors.129const COLORS = ["#2a78d6","#eb6834","#1baf7a","#eda100","#e87ba4","#008300","#4a3aa7","#e34948"];130const dark = () => window.matchMedia("(prefers-color-scheme: dark)").matches;131 132let ROWS = [], CONFIGS = [], COLOR = {};133 134const fmtUsd = v => "$" + v.toFixed(4);135const fmtMs  = v => v >= 1000 ? (v/1000).toFixed(2) + " s" : v.toFixed(0) + " ms";136 137function admissible(budget) {138  // Best point per config under the budget: highest throughput that still139  // meets it. Throughput is maximised rather than latency minimised because140  // for a fixed SLA the cheapest config is the one serving the most tokens141  // per GPU-hour while still inside it.142  const best = {};143  for (const r of ROWS) {144    if (r.ttft_p95_ms > budget) continue;145    if (!best[r.config_id] || r.throughput_tokens_s > best[r.config_id].throughput_tokens_s) {146      best[r.config_id] = r;147    }148  }149  return best;150}151 152function renderVerdict(budget) {153  const best = admissible(budget);154  const picks = Object.values(best);155  const el = document.getElementById("verdict");156  if (!picks.length) {157    el.innerHTML = "<strong>No configuration meets this budget.</strong> " +158      "The tightest measured p95 TTFT is " +159      fmtMs(Math.min(...ROWS.map(r => r.ttft_p95_ms))) + ".";160    return;161  }162  picks.sort((a, b) => a.cost_per_1m_usd - b.cost_per_1m_usd);163  const w = picks[0];164  const bf = best["vllm-bf16"];165  let vs = "";166  if (bf && bf.config_id !== w.config_id) {167    const saving = (1 - w.cost_per_1m_usd / bf.cost_per_1m_usd) * 100;168    vs = ` &mdash; ${saving.toFixed(0)}% cheaper than BF16, which serves ` +169         `${bf.throughput_tokens_s.toFixed(0)} tok/s at ${fmtUsd(bf.cost_per_1m_usd)}.`;170  }171  el.innerHTML =172    `Cheapest configuration meeting <strong>${budget} ms</strong>: ` +173    `<strong><code>${w.config_id}</code></strong> at ${w.rate_rps} req/s, ` +174    `serving <strong>${w.throughput_tokens_s.toFixed(0)} tok/s</strong> for ` +175    `<strong>${fmtUsd(w.cost_per_1m_usd)}</strong> per million output tokens${vs}` +176    `<br><span style="color:var(--muted)">${picks.length} of ${CONFIGS.length} ` +177    `configurations admissible at this budget.</span>`;178}179 180function renderTable(budget) {181  const best = admissible(budget);182  const cheapest = Object.values(best).sort((a,b) => a.cost_per_1m_usd - b.cost_per_1m_usd)[0];183  const body = document.getElementById("tbl").querySelector("tbody");184  body.innerHTML = ROWS.map(r => {185    const ok = r.ttft_p95_ms <= budget;186    const isBest = cheapest && r.config_id === cheapest.config_id && r.rate_rps === cheapest.rate_rps;187    return `<tr class="${isBest ? "best" : (ok ? "" : "out")}">188      <td><code>${r.config_id}</code></td>189      <td>${r.rate_rps} rps</td>190      <td>${r.throughput_tokens_s.toFixed(0)} ± ${r.throughput_std.toFixed(0)}</td>191      <td>${fmtMs(r.ttft_p95_ms)}</td>192      <td>${r.tpot_p95_ms.toFixed(1)} ms</td>193      <td>${fmtUsd(r.cost_per_1m_usd)}</td>194      <td>${r.repeats}</td>195      <td><span class="tag ${ok ? "in" : "out"}">${ok ? "meets" : "over"}</span></td>196    </tr>`;197  }).join("");198}199 200function renderChart(budget) {201  // Fail in the open. The chart library is the page's one external202  // dependency; if it does not load, an empty panel looks like a page with203  // no data, which is worse than a page that says what is wrong.204  if (typeof Plotly === "undefined") {205    document.getElementById("chart").innerHTML =206      '<p class="note" style="padding:32px 0;text-align:center">' +207      "The chart library failed to load from its CDN. The table and the budget " +208      "verdict above are computed locally and remain correct.</p>";209    return;210  }211  const muted = dark() ? "#a8a69e" : "#52514e";212  const grid  = dark() ? "#33322c" : "#e3e2df";213  const traces = CONFIGS.map(cfg => {214    const pts = ROWS.filter(r => r.config_id === cfg)215                    .sort((a,b) => a.throughput_tokens_s - b.throughput_tokens_s);216    return {217      x: pts.map(p => p.throughput_tokens_s),218      y: pts.map(p => p.ttft_p95_ms),219      name: cfg,220      type: "scatter",221      mode: "lines+markers",222      line: { color: COLOR[cfg], width: 2 },223      marker: { size: 7, color: COLOR[cfg] },224      hovertemplate:225        "<b>%{fullData.name}</b><br>%{x:.0f} tok/s<br>TTFT p95 %{y:.0f} ms" +226        "<extra></extra>"227    };228  });229  const layout = {230    margin: { l: 62, r: 18, t: 10, b: 52 },231    paper_bgcolor: "rgba(0,0,0,0)",232    plot_bgcolor: "rgba(0,0,0,0)",233    font: { color: muted, size: 12 },234    xaxis: { title: "Output token throughput (tokens/sec)", gridcolor: grid, zeroline: false },235    yaxis: { title: "p95 time to first token (ms)", type: "log", gridcolor: grid, zeroline: false },236    legend: { orientation: "h", y: -0.22, font: { size: 11.5 } },237    shapes: [{238      type: "line", xref: "paper", x0: 0, x1: 1, y0: budget, y1: budget,239      line: { color: muted, width: 1.4, dash: "dash" }240    }],241    annotations: [{242      xref: "paper", x: 1, y: Math.log10(budget), xanchor: "right", yanchor: "bottom",243      text: `SLA ${budget} ms`, showarrow: false, font: { size: 11, color: muted }244    }]245  };246  Plotly.react("chart", traces, layout, { displayModeBar: false, responsive: true });247}248 249function refresh() {250  const budget = Number(document.getElementById("sla").value);251  document.getElementById("slaVal").textContent = budget + " ms";252  renderVerdict(budget);253  renderTable(budget);254  renderChart(budget);255}256 257fetch("results.json")258  .then(r => r.json())259  .then(data => {260    ROWS = data;261    CONFIGS = [...new Set(data.map(r => r.config_id))].sort();262    CONFIGS.forEach((c, i) => { COLOR[c] = COLORS[i % COLORS.length]; });263    // Counts come from the data the page loaded, never from typed text: a264    // typed "168 measurements" survived two releases after it stopped being true.265    const repeats = data.reduce((n, r) => n + r.repeats, 0);266    document.getElementById("lede").innerHTML =267      `${data.length} reportable operating points from ${repeats} runs &middot; ` +268      `${CONFIGS.length} configurations &middot; Llama-3.1-8B-Instruct &middot; ` +269      "open-loop Poisson arrivals, clocks locked, kernels asserted from engine logs.";270    // The price is recovered from any row -- cost = price / (tokens/s x 3600) x 1e6 --271    // so the note cannot drift from the price the costs were computed with.272    const r0 = data[0];273    const price = r0.cost_per_1m_usd * r0.throughput_tokens_s * 3600 / 1e6;274    document.getElementById("price").textContent = "$" + price.toFixed(2);275    document.getElementById("sla").addEventListener("input", refresh);276    window.matchMedia("(prefers-color-scheme: dark)")277          .addEventListener("change", () => renderChart(Number(document.getElementById("sla").value)));278    refresh();279  })280  .catch(err => {281    document.getElementById("verdict").textContent =282      "Could not load results.json: " + err.message;283  });284</script>285</body>286</html>287